generated from john/python-template
Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
012dc15042 | ||
|
|
66e2dce465 | ||
|
|
597be2691c | ||
|
|
4e8c562f92 | ||
|
|
0b63b53f53 | ||
|
|
6a3ee26733 | ||
|
|
97b3d0fd62 | ||
|
|
7b9715b3f1 | ||
|
|
3e418a0889 | ||
|
|
2ccea77520 |
+2
-2
@@ -49,8 +49,8 @@ PROMPT_DIR="./prompts"
|
||||
# --- worker reliability ---
|
||||
WORKER_MAX_RETRIES=0
|
||||
WORKER_RETRY_BACKOFF_SECONDS=0
|
||||
# WORKER_PROVIDER_TIMEOUT_SECONDS=[0-20]
|
||||
WORKER_PROVIDER_TIMEOUT_SECONDS=20
|
||||
# WORKER_PROVIDER_TIMEOUT_SECONDS=180
|
||||
WORKER_PROVIDER_TIMEOUT_SECONDS=180
|
||||
WORKER_MIN_TRANSCRIPTION_CHARS=0
|
||||
WORKER_MIN_TRANSCRIPTION_LINES=0
|
||||
WORKER_FAIL_ON_FINISH_REASON_LENGTH=false
|
||||
|
||||
@@ -7,7 +7,7 @@ applyTo: 'src/transcription/services/*.py'
|
||||
|
||||
## Structure
|
||||
|
||||
- Project core data models defined in [models](../../src/transcription/models.py)
|
||||
- Project core data models defined in [models](../../src/transcription/db/models.py)
|
||||
- 1 service class per data model
|
||||
- Only services directly interact with the database, and only through async methods
|
||||
- Services are completely independent of one another. Any operation that needs to use more than a single service, which is most of them, needs to have a separate orchestration function.
|
||||
@@ -15,7 +15,7 @@ applyTo: 'src/transcription/services/*.py'
|
||||
## Error Handling
|
||||
|
||||
- Service-specific errors defined at the top of the respective module and inherit from `AppError`
|
||||
- Use a context manager for large `try/except` blocks like in [transcription](../../src/transcription/services/transcription.py)
|
||||
- Use a context manager for large `try/except` blocks like `handle_transcription_errors` in [sources](../../src/transcription/services/sources.py)
|
||||
|
||||
## Checklist
|
||||
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# Quality gate for V4.6 [HIGH-06]. Both hooks are blocking: a regression in
|
||||
# `ruff check` or `ty check` fails the commit.
|
||||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: ruff
|
||||
name: ruff check
|
||||
entry: ruff check
|
||||
language: system
|
||||
types_or: [python, pyi]
|
||||
require_serial: true
|
||||
- id: ty
|
||||
name: ty check
|
||||
entry: ty check
|
||||
language: system
|
||||
types_or: [python, pyi]
|
||||
pass_filenames: false
|
||||
require_serial: true
|
||||
+4
-1
@@ -7,7 +7,10 @@ services:
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
DATABASE_URL: sqlite:////app/data/transcription.db
|
||||
# Database configuration uses nested settings names (env_nested_delimiter="__").
|
||||
# DATABASE_URL is NOT read by the application and must not be used here.
|
||||
DATABASE__DRIVER: sqlite
|
||||
DATABASE__PATH: /app/data/transcription.db
|
||||
UPLOAD_DIR: /app/uploads
|
||||
PROMPT_DIR: /app/prompts
|
||||
ports:
|
||||
|
||||
@@ -41,6 +41,7 @@ dev = [
|
||||
[tool.pytest.ini_options]
|
||||
addopts = "--strict-markers -q"
|
||||
asyncio_mode = "strict"
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
filterwarnings = [
|
||||
"error:coroutine .* was never awaited:RuntimeWarning",
|
||||
]
|
||||
|
||||
@@ -12,7 +12,7 @@ from fastapi import HTTPException
|
||||
from fastapi import Request
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from transcription.services.sources import SOURCE_MIME_TYPES
|
||||
from transcription.services.source_media import SOURCE_MIME_TYPES
|
||||
from transcription.services.sources import SourceService
|
||||
|
||||
router = APIRouter(prefix="/api/v4", tags=["v4-print"])
|
||||
|
||||
@@ -25,10 +25,6 @@ from .db import create_all
|
||||
from .db import dispose_database_runtime
|
||||
from .db import initialize_database_runtime
|
||||
from .services import ServiceBundle
|
||||
from .services.documents import DocumentService
|
||||
from .services.jobs import JobService
|
||||
from .services.people import PeopleService
|
||||
from .services.sources import SourceService
|
||||
from .ui import register_pages
|
||||
from .worker import worker_consumer_lifespan
|
||||
|
||||
@@ -42,12 +38,7 @@ async def _lifespan(app: FastAPI):
|
||||
app.state.settings = settings
|
||||
app.state.runtime = initialize_database_runtime(settings=settings)
|
||||
session_factory = app.state.runtime.session_factory
|
||||
app.state.services = ServiceBundle(
|
||||
documents=DocumentService(session_factory=session_factory, settings=settings),
|
||||
sources=SourceService(session_factory=session_factory, settings=settings),
|
||||
jobs=JobService(session_factory=session_factory, settings=settings),
|
||||
people=PeopleService(session_factory=session_factory, settings=settings),
|
||||
)
|
||||
app.state.services = ServiceBundle.from_session_factory(session_factory, settings=settings)
|
||||
|
||||
if settings.should_bootstrap_schema:
|
||||
await create_all(engine=app.state.runtime.engine)
|
||||
@@ -78,8 +69,7 @@ async def _recover_stale_processing_jobs(app: FastAPI) -> None:
|
||||
"""
|
||||
settings = app.state.settings
|
||||
stale_before = datetime.now(UTC) - timedelta(seconds=settings.worker_provider_timeout_seconds)
|
||||
job_service = JobService(session_factory=app.state.runtime.session_factory)
|
||||
recovered = await job_service.requeue_stale_processing_jobs(stale_before=stale_before)
|
||||
recovered = await app.state.services.jobs.requeue_stale_processing_jobs(stale_before=stale_before)
|
||||
if recovered > 0:
|
||||
logger.warning("Recovered %s stale processing job(s) at startup", recovered)
|
||||
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
"""Helpers for accessing lifespan-owned application state resources."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import FastAPI
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from transcription.db.runtime import DatabaseRuntime
|
||||
from transcription.db.session import get_session_factory
|
||||
from transcription.worker import WorkerNotifier
|
||||
from transcription.worker import resolve_worker_notifier
|
||||
|
||||
|
||||
def resolve_database_runtime(state: object) -> DatabaseRuntime | None:
|
||||
"""Return database runtime from app-like state objects when available."""
|
||||
runtime = getattr(state, "runtime", None)
|
||||
return runtime if isinstance(runtime, DatabaseRuntime) else None
|
||||
|
||||
|
||||
def require_database_runtime(state: object) -> DatabaseRuntime:
|
||||
"""Return database runtime or raise when app lifespan has not initialized it."""
|
||||
runtime = resolve_database_runtime(state)
|
||||
if runtime is None:
|
||||
raise RuntimeError("Database runtime is not initialized on application state")
|
||||
return runtime
|
||||
|
||||
|
||||
def resolve_session_factory(state: object) -> async_sessionmaker[AsyncSession]:
|
||||
"""Return DB session factory from state when available, otherwise shared runtime."""
|
||||
runtime = resolve_database_runtime(state)
|
||||
if runtime is not None:
|
||||
return runtime.session_factory
|
||||
return get_session_factory()
|
||||
|
||||
|
||||
def get_worker_notifier(app: FastAPI) -> WorkerNotifier:
|
||||
"""Return app worker notifier, or a no-op fallback when unavailable."""
|
||||
return resolve_worker_notifier(app.state)
|
||||
+38
-14
@@ -102,12 +102,14 @@ class Settings(BaseSettings):
|
||||
upload_dir: Path = Path("./uploads")
|
||||
prompt_dir: Path = Path("./prompts")
|
||||
artifact_dir: Path = Path("./data/artifacts")
|
||||
homepage_dir: Path = Path("./data/homepage")
|
||||
artifact_inline_threshold_bytes: int = Field(default=1_048_576, ge=1)
|
||||
|
||||
# --- worker reliability ---
|
||||
worker_max_retries: int = Field(default=0, ge=0)
|
||||
worker_retry_backoff_seconds: float = Field(default=0.0, ge=0.0)
|
||||
worker_provider_timeout_seconds: float = Field(default=20.0, gt=0.0, le=20.0)
|
||||
# Bounded only from below. Vision transcription of a dense page routinely runs
|
||||
# well past twenty seconds, so an upper cap here would silently fail real work.
|
||||
worker_provider_timeout_seconds: float = Field(default=180.0, gt=0.0)
|
||||
worker_min_transcription_chars: int = Field(default=0, ge=0)
|
||||
worker_min_transcription_lines: int = Field(default=0, ge=0)
|
||||
worker_fail_on_finish_reason_length: bool = False
|
||||
@@ -121,21 +123,43 @@ class Settings(BaseSettings):
|
||||
raise ValueError("PROVIDER_MODELS must contain at least one model")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def normalize_provider_models(self) -> "Settings":
|
||||
"""Build the immutable model selector with the configured default first."""
|
||||
configured = self.provider_models
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def normalize_provider_models(cls, data: object) -> object:
|
||||
"""Build the immutable model selector with the configured default first.
|
||||
|
||||
This runs before field validation so the derived value is produced by
|
||||
normal construction rather than by mutating a frozen instance.
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
default_model = data.get("provider_model") or DEFAULT_PROVIDER_MODEL
|
||||
if not isinstance(default_model, str):
|
||||
return data
|
||||
default_model = default_model.strip()
|
||||
|
||||
configured = data.get("provider_models")
|
||||
if configured is None:
|
||||
configured = ()
|
||||
elif isinstance(configured, str):
|
||||
# Left as-is so the field validator can report the malformed value.
|
||||
return {**data, "provider_model": default_model}
|
||||
elif not isinstance(configured, (list, tuple)):
|
||||
return {**data, "provider_model": default_model}
|
||||
elif not configured:
|
||||
# Preserved so validate_provider_models_input can reject it.
|
||||
return {**data, "provider_model": default_model}
|
||||
|
||||
default_model = self.provider_model or DEFAULT_PROVIDER_MODEL
|
||||
object.__setattr__(self, "provider_model", default_model)
|
||||
ordered = (default_model, *configured)
|
||||
deduplicated: list[str] = []
|
||||
for model in ordered:
|
||||
for model in (default_model, *configured):
|
||||
if not isinstance(model, str):
|
||||
return {**data, "provider_model": default_model}
|
||||
normalized = model.strip()
|
||||
if normalized not in deduplicated:
|
||||
deduplicated.append(normalized)
|
||||
object.__setattr__(self, "provider_models", tuple(deduplicated))
|
||||
return self
|
||||
|
||||
return {**data, "provider_model": default_model, "provider_models": tuple(deduplicated)}
|
||||
|
||||
@property
|
||||
def should_bootstrap_schema(self) -> bool:
|
||||
@@ -148,13 +172,13 @@ class Settings(BaseSettings):
|
||||
@cache
|
||||
def get_settings(**kwargs: Any) -> Settings:
|
||||
"""Load cached settings without reading process CLI arguments."""
|
||||
return Settings(_cli_parse_args=False, **kwargs) # pyright: ignore[reportCallIssue]
|
||||
return Settings(_cli_parse_args=False, **kwargs)
|
||||
|
||||
|
||||
def parse_cli_settings(args: Sequence[str] | None = None) -> Settings:
|
||||
"""Load settings with CLI arguments at the executable boundary."""
|
||||
cli_args = True if args is None else list(args)
|
||||
return Settings(_cli_parse_args=cli_args) # pyright: ignore[reportCallIssue]
|
||||
return Settings(_cli_parse_args=cli_args)
|
||||
|
||||
|
||||
LOGGING_CONFIG: dict[str, Any] = {
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
from .operations import create_all
|
||||
from .operations import upgrade_schema
|
||||
from .runtime import dispose_database_runtime
|
||||
from .runtime import initialize_database_runtime
|
||||
from .session import session_scope
|
||||
@@ -11,5 +10,4 @@ __all__ = [
|
||||
"initialize_database_runtime",
|
||||
"session_scope",
|
||||
"transaction_scope",
|
||||
"upgrade_schema",
|
||||
]
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
from functools import cache
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import URL
|
||||
@@ -33,26 +32,53 @@ def get_database_url(settings: Settings) -> str:
|
||||
|
||||
def resolve_engine(settings: Settings | None = None) -> AsyncEngine:
|
||||
active_settings = settings or get_settings()
|
||||
return get_engine(get_database_url(active_settings))
|
||||
return get_engine(
|
||||
get_database_url(active_settings),
|
||||
sqlite_check_same_thread=active_settings.sqlite_check_same_thread,
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def get_engine(database_url: str) -> AsyncEngine:
|
||||
_ENGINES: dict[str, AsyncEngine] = {}
|
||||
|
||||
|
||||
def _create_engine(database_url: str, *, sqlite_check_same_thread: bool) -> AsyncEngine:
|
||||
kwargs: dict[str, Any] = {"echo": False, "pool_pre_ping": True}
|
||||
if database_url.startswith("sqlite"):
|
||||
kwargs["connect_args"] = {"check_same_thread": False}
|
||||
kwargs["connect_args"] = {"check_same_thread": sqlite_check_same_thread}
|
||||
if ":memory:" in database_url:
|
||||
kwargs["poolclass"] = StaticPool
|
||||
|
||||
return create_async_engine(database_url, **kwargs)
|
||||
|
||||
|
||||
def get_engine(database_url: str, *, sqlite_check_same_thread: bool = False) -> AsyncEngine:
|
||||
"""Return the process-wide engine for ``database_url``, creating it on first use.
|
||||
|
||||
Engines are registered per URL so that disposing one leaves every other
|
||||
database untouched.
|
||||
"""
|
||||
engine = _ENGINES.get(database_url)
|
||||
if engine is None:
|
||||
engine = _create_engine(database_url, sqlite_check_same_thread=sqlite_check_same_thread)
|
||||
_ENGINES[database_url] = engine
|
||||
return engine
|
||||
|
||||
|
||||
async def dispose_engine(database_url: str) -> None:
|
||||
engine = get_engine(database_url)
|
||||
try:
|
||||
"""Dispose and unregister the engine for ``database_url`` only.
|
||||
|
||||
Unknown URLs are a no-op rather than provoking the creation of an engine
|
||||
purely so that it can be thrown away.
|
||||
"""
|
||||
engine = _ENGINES.pop(database_url, None)
|
||||
if engine is not None:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def dispose_all_engines() -> None:
|
||||
while _ENGINES:
|
||||
_, engine = _ENGINES.popitem()
|
||||
await engine.dispose()
|
||||
finally:
|
||||
get_engine.cache_clear()
|
||||
|
||||
|
||||
async def refresh_engine(database_url: str) -> AsyncEngine:
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Typed loader-option wrappers for SQLModel relationship attributes.
|
||||
|
||||
SQLModel declares relationships with their runtime Python type, so
|
||||
``Document.jobs`` is annotated ``list[Job]`` even though at runtime it is an
|
||||
``InstrumentedAttribute``. SQLAlchemy's loader options are typed against
|
||||
``QueryableAttribute``, so every eager-load call site reads as a type error to a
|
||||
static checker even though the code is correct.
|
||||
|
||||
These wrappers put that reinterpretation in one documented place instead of
|
||||
scattering a suppression comment across every eager-load call. Import
|
||||
``selectinload`` and ``defer`` from here rather than from ``sqlalchemy.orm``.
|
||||
|
||||
Multi-level eager loads must keep using the chained form --
|
||||
``selectinload(A.b).selectinload(orm_attribute(B.c))`` -- and not the varargs
|
||||
form ``selectinload(A.b, B.c)``. The two produce the same loader path, but
|
||||
varargs applies the selectin strategy only to the last element while the
|
||||
intermediate falls back to its default strategy. Every relationship here
|
||||
declares ``lazy="raise"``, so the varargs form raises at render time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from typing import cast
|
||||
|
||||
from sqlalchemy.orm import defer as _defer
|
||||
from sqlalchemy.orm import selectinload as _selectinload
|
||||
from sqlalchemy.orm.attributes import QueryableAttribute
|
||||
from sqlalchemy.orm.strategy_options import _AbstractLoad
|
||||
|
||||
|
||||
def orm_attribute(attribute: object) -> QueryableAttribute[Any]:
|
||||
"""Reinterpret a SQLModel relationship or field as its ORM descriptor."""
|
||||
return cast("QueryableAttribute[Any]", attribute)
|
||||
|
||||
|
||||
def selectinload(*keys: object) -> _AbstractLoad:
|
||||
"""``sqlalchemy.orm.selectinload`` accepting SQLModel-annotated attributes."""
|
||||
return _selectinload(*(orm_attribute(key) for key in keys))
|
||||
|
||||
|
||||
def defer(*keys: object, raiseload: bool = False) -> _AbstractLoad:
|
||||
"""``sqlalchemy.orm.defer`` accepting SQLModel-annotated attributes."""
|
||||
first, *rest = (orm_attribute(key) for key in keys)
|
||||
return _defer(first, *rest, raiseload=raiseload)
|
||||
+103
-52
@@ -4,6 +4,7 @@ from datetime import UTC
|
||||
from datetime import date
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
from typing import Optional
|
||||
from uuid import UUID
|
||||
from uuid import uuid4
|
||||
@@ -14,16 +15,38 @@ from sqlalchemy import BigInteger
|
||||
from sqlalchemy import CheckConstraint
|
||||
from sqlalchemy import Column
|
||||
from sqlalchemy import Enum as SAEnum
|
||||
from sqlalchemy import ForeignKey
|
||||
from sqlalchemy import Index
|
||||
from sqlalchemy import LargeBinary
|
||||
from sqlalchemy import UniqueConstraint
|
||||
from sqlalchemy import Uuid
|
||||
from sqlalchemy import inspect as sqlalchemy_inspect
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm.exc import DetachedInstanceError
|
||||
from sqlalchemy.exc import NoInspectionAvailable
|
||||
from sqlalchemy.orm.state import InstanceState
|
||||
from sqlalchemy.types import TypeDecorator
|
||||
from sqlmodel import Field
|
||||
from sqlmodel import Relationship
|
||||
from sqlmodel import SQLModel
|
||||
|
||||
|
||||
def _loaded_attribute(instance: object, attribute: str) -> Any | None:
|
||||
"""Return ``attribute`` only when it is already loaded on ``instance``.
|
||||
|
||||
Relationships in this module declare ``lazy="raise"``, so reading an
|
||||
unloaded attribute is an error rather than a silent query. Callers that
|
||||
render optional detail use this to distinguish "not loaded" from "absent"
|
||||
without catching exceptions indiscriminately.
|
||||
"""
|
||||
try:
|
||||
state: InstanceState[Any] = sqlalchemy_inspect(instance, raiseerr=True)
|
||||
except NoInspectionAvailable:
|
||||
return None
|
||||
if attribute in state.unloaded:
|
||||
return None
|
||||
return state.dict.get(attribute)
|
||||
|
||||
|
||||
class JSONBCompat(TypeDecorator):
|
||||
"""JSONB for PostgreSQL and JSON for SQLite/testing backends."""
|
||||
|
||||
@@ -66,10 +89,13 @@ class DocumentType(SQLModel, table=True):
|
||||
normalized_label: str = Field(index=True, unique=True)
|
||||
is_active: bool = True
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
updated_at: datetime = Field(
|
||||
default_factory=lambda: datetime.now(UTC),
|
||||
sa_column_kwargs={"onupdate": lambda: datetime.now(UTC)},
|
||||
)
|
||||
|
||||
documents: list["Document"] = Relationship(
|
||||
back_populates="document_type_ref", sa_relationship_kwargs={"lazy": "selectin"}
|
||||
back_populates="document_type_ref", sa_relationship_kwargs={"lazy": "raise"}
|
||||
)
|
||||
|
||||
|
||||
@@ -84,10 +110,13 @@ class PersonRole(SQLModel, table=True):
|
||||
normalized_label: str = Field(index=True, unique=True)
|
||||
is_active: bool = True
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
updated_at: datetime = Field(
|
||||
default_factory=lambda: datetime.now(UTC),
|
||||
sa_column_kwargs={"onupdate": lambda: datetime.now(UTC)},
|
||||
)
|
||||
|
||||
document_people: list["DocumentPerson"] = Relationship(
|
||||
back_populates="role_ref", sa_relationship_kwargs={"lazy": "selectin"}
|
||||
back_populates="role_ref", sa_relationship_kwargs={"lazy": "raise"}
|
||||
)
|
||||
|
||||
|
||||
@@ -96,22 +125,25 @@ class Document(SQLModel, table=True):
|
||||
|
||||
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
||||
name: str
|
||||
document_type_id: UUID | None = Field(default=None, foreign_key="document_type.id")
|
||||
document_type_id: UUID | None = Field(default=None, foreign_key="document_type.id", index=True)
|
||||
document_date: date | None = None
|
||||
document_date_raw: str | None = None
|
||||
location_created: str | None = None
|
||||
notes: str | None = None
|
||||
archive_identifier: str | None = None
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
updated_at: datetime = Field(
|
||||
default_factory=lambda: datetime.now(UTC),
|
||||
sa_column_kwargs={"onupdate": lambda: datetime.now(UTC)},
|
||||
)
|
||||
|
||||
jobs: list["Job"] = Relationship(back_populates="document", sa_relationship_kwargs={"lazy": "selectin"})
|
||||
sources: list["Source"] = Relationship(back_populates="document", sa_relationship_kwargs={"lazy": "selectin"})
|
||||
jobs: list["Job"] = Relationship(back_populates="document", sa_relationship_kwargs={"lazy": "raise"})
|
||||
sources: list["Source"] = Relationship(back_populates="document", sa_relationship_kwargs={"lazy": "raise"})
|
||||
document_people: list["DocumentPerson"] = Relationship(
|
||||
back_populates="document", sa_relationship_kwargs={"lazy": "selectin"}
|
||||
back_populates="document", sa_relationship_kwargs={"lazy": "raise"}
|
||||
)
|
||||
document_type_ref: Optional["DocumentType"] = Relationship(
|
||||
back_populates="documents", sa_relationship_kwargs={"lazy": "selectin"}
|
||||
back_populates="documents", sa_relationship_kwargs={"lazy": "raise"}
|
||||
)
|
||||
|
||||
|
||||
@@ -136,10 +168,13 @@ class Person(SQLModel, table=True):
|
||||
sa_column=Column("metadata", JSONBCompat(), nullable=True),
|
||||
)
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
updated_at: datetime = Field(
|
||||
default_factory=lambda: datetime.now(UTC),
|
||||
sa_column_kwargs={"onupdate": lambda: datetime.now(UTC)},
|
||||
)
|
||||
|
||||
document_people: list["DocumentPerson"] = Relationship(
|
||||
back_populates="person", sa_relationship_kwargs={"lazy": "selectin"}
|
||||
back_populates="person", sa_relationship_kwargs={"lazy": "raise"}
|
||||
)
|
||||
|
||||
|
||||
@@ -149,30 +184,35 @@ class DocumentPerson(SQLModel, table=True):
|
||||
__tablename__ = "document_person"
|
||||
|
||||
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
||||
document_id: UUID = Field(foreign_key="document.id")
|
||||
person_id: UUID = Field(foreign_key="person.id")
|
||||
role_id: UUID = Field(foreign_key="person_role.id")
|
||||
document_id: UUID = Field(foreign_key="document.id", index=True)
|
||||
person_id: UUID = Field(foreign_key="person.id", index=True)
|
||||
role_id: UUID = Field(foreign_key="person_role.id", index=True)
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
updated_at: datetime = Field(
|
||||
default_factory=lambda: datetime.now(UTC),
|
||||
sa_column_kwargs={"onupdate": lambda: datetime.now(UTC)},
|
||||
)
|
||||
|
||||
__table_args__ = (UniqueConstraint("document_id", "person_id", name="uq_document_person"),)
|
||||
|
||||
document: Optional["Document"] = Relationship(
|
||||
back_populates="document_people", sa_relationship_kwargs={"lazy": "selectin"}
|
||||
back_populates="document_people", sa_relationship_kwargs={"lazy": "raise"}
|
||||
)
|
||||
person: Optional["Person"] = Relationship(
|
||||
back_populates="document_people", sa_relationship_kwargs={"lazy": "selectin"}
|
||||
back_populates="document_people", sa_relationship_kwargs={"lazy": "raise"}
|
||||
)
|
||||
role_ref: Optional["PersonRole"] = Relationship(
|
||||
back_populates="document_people", sa_relationship_kwargs={"lazy": "selectin"}
|
||||
back_populates="document_people", sa_relationship_kwargs={"lazy": "raise"}
|
||||
)
|
||||
|
||||
|
||||
class Job(SQLModel, table=True):
|
||||
"""A transcription job tied to a single document."""
|
||||
|
||||
__table_args__ = (Index("ix_job_status_date_created", "status", "date_created"),)
|
||||
|
||||
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
||||
document_id: UUID = Field(foreign_key="document.id")
|
||||
document_id: UUID = Field(foreign_key="document.id", index=True)
|
||||
status: JobStatus = Field(
|
||||
default=JobStatus.QUEUED,
|
||||
sa_column=Column(
|
||||
@@ -198,7 +238,10 @@ class Job(SQLModel, table=True):
|
||||
),
|
||||
)
|
||||
date_created: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
date_updated: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
date_updated: datetime = Field(
|
||||
default_factory=lambda: datetime.now(UTC),
|
||||
sa_column_kwargs={"onupdate": lambda: datetime.now(UTC)},
|
||||
)
|
||||
provider: str | None = None
|
||||
model: str | None = None
|
||||
prompt_name: str | None = None
|
||||
@@ -208,25 +251,20 @@ class Job(SQLModel, table=True):
|
||||
temperature: float | None = None
|
||||
top_p: float | None = None
|
||||
|
||||
document: Optional["Document"] = Relationship(back_populates="jobs", sa_relationship_kwargs={"lazy": "selectin"})
|
||||
job_sources: list["JobSource"] = Relationship(back_populates="job", sa_relationship_kwargs={"lazy": "selectin"})
|
||||
document: Optional["Document"] = Relationship(back_populates="jobs", sa_relationship_kwargs={"lazy": "raise"})
|
||||
job_sources: list["JobSource"] = Relationship(back_populates="job", sa_relationship_kwargs={"lazy": "raise"})
|
||||
|
||||
@property
|
||||
def filename(self) -> str:
|
||||
"""Return the filename of the associated source, when available."""
|
||||
if not self.job_sources:
|
||||
return "unknown"
|
||||
|
||||
for job_source in self.job_sources:
|
||||
source = job_source.__dict__.get("source")
|
||||
if source is None:
|
||||
try:
|
||||
source = job_source.source
|
||||
except DetachedInstanceError:
|
||||
source = None
|
||||
except Exception: # noqa: BLE001
|
||||
source = None
|
||||
"""Return the filename of the first loaded source, when available.
|
||||
|
||||
Relationships on this model use ``lazy="raise"``, so this deliberately
|
||||
inspects load state rather than triggering (or swallowing) a lazy load:
|
||||
a read model that did not eager-load its sources gets "unknown" instead
|
||||
of an unhandled error, and genuine errors are no longer hidden.
|
||||
"""
|
||||
for job_source in _loaded_attribute(self, "job_sources") or ():
|
||||
source = _loaded_attribute(job_source, "source")
|
||||
if source is not None:
|
||||
return source.filename
|
||||
|
||||
@@ -235,10 +273,7 @@ class Job(SQLModel, table=True):
|
||||
@property
|
||||
def error_detail(self) -> str | None:
|
||||
"""Return the first available source-level error detail for the job."""
|
||||
if not self.job_sources:
|
||||
return None
|
||||
|
||||
for job_source in self.job_sources:
|
||||
for job_source in _loaded_attribute(self, "job_sources") or ():
|
||||
if job_source.error_detail:
|
||||
return job_source.error_detail
|
||||
|
||||
@@ -249,7 +284,7 @@ class Source(SQLModel, table=True):
|
||||
"""A document source image or PDF page."""
|
||||
|
||||
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
||||
document_id: UUID = Field(foreign_key="document.id")
|
||||
document_id: UUID = Field(foreign_key="document.id", index=True)
|
||||
page_number: int = Field(default=1, ge=1)
|
||||
upload_name: str
|
||||
filename: str
|
||||
@@ -259,8 +294,18 @@ class Source(SQLModel, table=True):
|
||||
raw_transcription: str | None = None
|
||||
preferred_execution_attempt_id: UUID | None = Field(
|
||||
default=None,
|
||||
foreign_key="execution_attempt.id",
|
||||
sa_column=Column(
|
||||
Uuid(),
|
||||
# use_alter breaks the source / job_source / execution_attempt cycle so
|
||||
# metadata.create_all can order table creation on every dialect.
|
||||
ForeignKey(
|
||||
"execution_attempt.id",
|
||||
use_alter=True,
|
||||
name="fk_source_preferred_execution_attempt_id",
|
||||
),
|
||||
nullable=True,
|
||||
index=True,
|
||||
),
|
||||
)
|
||||
revised_text: str | None = None
|
||||
date_uploaded: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
@@ -268,11 +313,11 @@ class Source(SQLModel, table=True):
|
||||
|
||||
document: Optional["Document"] = Relationship(
|
||||
back_populates="sources",
|
||||
sa_relationship_kwargs={"lazy": "selectin"},
|
||||
sa_relationship_kwargs={"lazy": "raise"},
|
||||
)
|
||||
job_sources: list["JobSource"] = Relationship(
|
||||
back_populates="source",
|
||||
sa_relationship_kwargs={"lazy": "selectin"},
|
||||
sa_relationship_kwargs={"lazy": "raise"},
|
||||
)
|
||||
processing_artifacts: list["ProcessingArtifact"] = Relationship(
|
||||
back_populates="source",
|
||||
@@ -310,8 +355,8 @@ class JobSource(SQLModel, table=True):
|
||||
__tablename__ = "job_source"
|
||||
|
||||
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
||||
job_id: UUID = Field(foreign_key="job.id")
|
||||
source_id: UUID = Field(foreign_key="source.id")
|
||||
job_id: UUID = Field(foreign_key="job.id", index=True)
|
||||
source_id: UUID = Field(foreign_key="source.id", index=True)
|
||||
status: JobSourceStatus = Field(
|
||||
default=JobSourceStatus.PENDING,
|
||||
sa_column=Column(
|
||||
@@ -329,8 +374,8 @@ class JobSource(SQLModel, table=True):
|
||||
error_detail: str | None = None
|
||||
executed_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
|
||||
job: Optional["Job"] = Relationship(back_populates="job_sources", sa_relationship_kwargs={"lazy": "selectin"})
|
||||
source: Optional["Source"] = Relationship(back_populates="job_sources", sa_relationship_kwargs={"lazy": "selectin"})
|
||||
job: Optional["Job"] = Relationship(back_populates="job_sources", sa_relationship_kwargs={"lazy": "raise"})
|
||||
source: Optional["Source"] = Relationship(back_populates="job_sources", sa_relationship_kwargs={"lazy": "raise"})
|
||||
execution_attempts: list["ExecutionAttempt"] = Relationship(
|
||||
back_populates="job_source",
|
||||
sa_relationship_kwargs={"lazy": "noload", "order_by": "ExecutionAttempt.attempt_number"},
|
||||
@@ -380,7 +425,9 @@ class ExecutionAttempt(SQLModel, table=True):
|
||||
duration_ms: int = Field(ge=0)
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
|
||||
job_source: Optional["JobSource"] = Relationship(back_populates="execution_attempts")
|
||||
job_source: Optional["JobSource"] = Relationship(
|
||||
back_populates="execution_attempts", sa_relationship_kwargs={"lazy": "raise"}
|
||||
)
|
||||
artifacts: list["ProcessingArtifact"] = Relationship(
|
||||
back_populates="execution_attempt", sa_relationship_kwargs={"lazy": "noload"}
|
||||
)
|
||||
@@ -416,5 +463,9 @@ class ProcessingArtifact(SQLModel, table=True):
|
||||
)
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
|
||||
execution_attempt: Optional["ExecutionAttempt"] = Relationship(back_populates="artifacts")
|
||||
source: Optional["Source"] = Relationship(back_populates="processing_artifacts")
|
||||
execution_attempt: Optional["ExecutionAttempt"] = Relationship(
|
||||
back_populates="artifacts", sa_relationship_kwargs={"lazy": "raise"}
|
||||
)
|
||||
source: Optional["Source"] = Relationship(
|
||||
back_populates="processing_artifacts", sa_relationship_kwargs={"lazy": "raise"}
|
||||
)
|
||||
|
||||
@@ -2,9 +2,6 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncConnection
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
from sqlmodel import SQLModel
|
||||
@@ -13,8 +10,6 @@ from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from .engine import resolve_engine
|
||||
from .models import DocumentType
|
||||
from .models import Job
|
||||
from .models import JobStatus
|
||||
from .models import PersonRole
|
||||
from .registries import BUILT_IN_DOCUMENT_TYPES
|
||||
from .registries import BUILT_IN_PERSON_ROLES
|
||||
@@ -30,85 +25,10 @@ async def create_all(*, engine: AsyncEngine | None = None) -> None:
|
||||
active_engine = engine or resolve_engine()
|
||||
async with active_engine.begin() as connection:
|
||||
await connection.run_sync(SQLModel.metadata.create_all)
|
||||
await _upgrade_person_family_search_id(connection)
|
||||
await _upgrade_v42_evidence_tables(connection)
|
||||
await _upgrade_v45_selection_columns(connection)
|
||||
await seed_registry_defaults(engine=active_engine)
|
||||
logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url)
|
||||
|
||||
|
||||
async def upgrade_schema(*, engine: AsyncEngine | None = None) -> None:
|
||||
"""Apply non-destructive additive upgrades to an existing schema."""
|
||||
active_engine = engine or resolve_engine()
|
||||
async with active_engine.begin() as connection:
|
||||
await _upgrade_person_family_search_id(connection)
|
||||
await _upgrade_v42_evidence_tables(connection)
|
||||
await _upgrade_v45_selection_columns(connection)
|
||||
|
||||
|
||||
async def _upgrade_v42_evidence_tables(connection: AsyncConnection) -> None:
|
||||
"""Create the additive V4.2 evidence tables without rewriting historical rows."""
|
||||
|
||||
def create_tables(sync_connection) -> None:
|
||||
SQLModel.metadata.tables["execution_attempt"].create(sync_connection, checkfirst=True)
|
||||
SQLModel.metadata.tables["processing_artifact"].create(sync_connection, checkfirst=True)
|
||||
|
||||
await connection.run_sync(create_tables)
|
||||
|
||||
|
||||
async def _upgrade_v45_selection_columns(connection: AsyncConnection) -> None:
|
||||
"""Add V4.5 purpose and preferred-attempt provenance columns."""
|
||||
|
||||
def inspect_columns(sync_connection) -> tuple[set[str], set[str]]:
|
||||
database = inspect(sync_connection)
|
||||
tables = set(database.get_table_names())
|
||||
job_columns = {column["name"] for column in database.get_columns("job")} if "job" in tables else set()
|
||||
source_columns = (
|
||||
{column["name"] for column in database.get_columns("source")} if "source" in tables else set()
|
||||
)
|
||||
return job_columns, source_columns
|
||||
|
||||
job_columns, source_columns = await connection.run_sync(inspect_columns)
|
||||
if job_columns and "purpose" not in job_columns:
|
||||
await connection.execute(
|
||||
text("ALTER TABLE job ADD COLUMN purpose VARCHAR NOT NULL DEFAULT 'transcription'")
|
||||
)
|
||||
if source_columns and "preferred_execution_attempt_id" not in source_columns:
|
||||
await connection.execute(text("ALTER TABLE source ADD COLUMN preferred_execution_attempt_id CHAR(32)"))
|
||||
await connection.execute(
|
||||
text(
|
||||
"CREATE INDEX IF NOT EXISTS ix_source_preferred_execution_attempt_id "
|
||||
"ON source (preferred_execution_attempt_id)"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _upgrade_person_family_search_id(connection: AsyncConnection) -> None:
|
||||
"""Add the nullable V4.1 FamilySearch field to an existing database."""
|
||||
|
||||
def inspect_person(sync_connection) -> tuple[bool, bool]:
|
||||
database = inspect(sync_connection)
|
||||
if "person" not in database.get_table_names():
|
||||
return False, False
|
||||
columns = {column["name"] for column in database.get_columns("person")}
|
||||
indexes = database.get_indexes("person")
|
||||
constraints = database.get_unique_constraints("person")
|
||||
has_unique_id = any(entry.get("column_names") == ["family_search_id"] for entry in [*indexes, *constraints])
|
||||
return "family_search_id" in columns, has_unique_id
|
||||
|
||||
has_column, has_unique_id = await connection.run_sync(inspect_person)
|
||||
if not has_column and not await connection.run_sync(
|
||||
lambda sync_connection: "person" in inspect(sync_connection).get_table_names()
|
||||
):
|
||||
return
|
||||
if not has_column:
|
||||
await connection.execute(text("ALTER TABLE person ADD COLUMN family_search_id VARCHAR"))
|
||||
if not has_unique_id:
|
||||
await connection.execute(
|
||||
text("CREATE UNIQUE INDEX IF NOT EXISTS ix_person_family_search_id ON person (family_search_id)")
|
||||
)
|
||||
|
||||
|
||||
async def seed_registry_defaults(*, engine: AsyncEngine | None = None) -> None:
|
||||
"""Seed default registry rows for role and document type taxonomies."""
|
||||
active_engine = engine or resolve_engine()
|
||||
@@ -138,14 +58,3 @@ async def seed_registry_defaults(*, engine: AsyncEngine | None = None) -> None:
|
||||
)
|
||||
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def get_next_queued_job(*, session: AsyncSession) -> Job | None:
|
||||
"""Get the next queued job, if any."""
|
||||
result = await session.exec(
|
||||
select(Job)
|
||||
.where(Job.status == JobStatus.QUEUED)
|
||||
.order_by(Job.date_created) # pyright: ignore[reportArgumentType]
|
||||
.limit(1)
|
||||
) # fmt: skip
|
||||
return result.first()
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
from functools import cache
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from sqlalchemy.ext.asyncio import AsyncSessionTransaction
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
@@ -17,13 +15,20 @@ from .engine import get_engine
|
||||
type SessionFactory = async_sessionmaker[AsyncSession]
|
||||
|
||||
|
||||
@cache
|
||||
_SESSION_FACTORIES: dict[str, SessionFactory] = {}
|
||||
|
||||
|
||||
def get_session_factory(database_url: str) -> SessionFactory:
|
||||
return async_sessionmaker(
|
||||
"""Return the process-wide session factory for ``database_url``."""
|
||||
factory = _SESSION_FACTORIES.get(database_url)
|
||||
if factory is None:
|
||||
factory = async_sessionmaker(
|
||||
bind=get_engine(database_url),
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
)
|
||||
_SESSION_FACTORIES[database_url] = factory
|
||||
return factory
|
||||
|
||||
|
||||
def resolve_session_factory(
|
||||
@@ -46,7 +51,8 @@ type SessionFactoryDep = Annotated[SessionFactory, Depends(resolve_session_facto
|
||||
|
||||
|
||||
async def dispose_session_factory(database_url: str) -> None:
|
||||
get_session_factory.cache_clear()
|
||||
"""Drop the session factory and engine for ``database_url`` only."""
|
||||
_SESSION_FACTORIES.pop(database_url, None)
|
||||
await dispose_engine(database_url)
|
||||
|
||||
|
||||
@@ -79,16 +85,12 @@ async def transaction_scope(
|
||||
settings: Settings | None = None,
|
||||
database_url: str | None = None,
|
||||
session_factory: SessionFactory | None = None,
|
||||
session: AsyncSession | AsyncSessionTransaction | None = None,
|
||||
) -> AsyncGenerator[AsyncSession | AsyncSessionTransaction]:
|
||||
match session:
|
||||
case AsyncSession() as async_session:
|
||||
if not async_session.in_transaction():
|
||||
session: AsyncSession | None = None,
|
||||
) -> AsyncGenerator[AsyncSession]:
|
||||
if session is not None:
|
||||
if not session.in_transaction():
|
||||
raise RuntimeError("A supplied session must have an active transaction")
|
||||
yield async_session
|
||||
return
|
||||
case AsyncSessionTransaction() as async_transaction:
|
||||
yield async_transaction
|
||||
yield session
|
||||
return
|
||||
|
||||
active_session_factory = session_factory or resolve_session_factory(
|
||||
@@ -99,7 +101,4 @@ async def transaction_scope(
|
||||
yield owned_session
|
||||
|
||||
|
||||
type TransactionScopeDep = Annotated[
|
||||
AsyncSession | AsyncSessionTransaction,
|
||||
Depends(transaction_scope),
|
||||
]
|
||||
type TransactionScopeDep = Annotated[AsyncSession, Depends(transaction_scope)]
|
||||
|
||||
@@ -102,6 +102,21 @@ class TranscriptionResult(BaseModel):
|
||||
class TranscriptionProvider(Protocol):
|
||||
"""Contract every transcription provider adapter must satisfy."""
|
||||
|
||||
@property
|
||||
def model(self) -> str:
|
||||
"""Return the resolved model slug this adapter will call."""
|
||||
...
|
||||
|
||||
@property
|
||||
def current_request_manifest(self) -> RequestManifest | None:
|
||||
"""Return the manifest for the most recent call, for failure evidence."""
|
||||
...
|
||||
|
||||
@property
|
||||
def current_transport_evidence(self) -> TransportEvidence | None:
|
||||
"""Return transport-level evidence for the most recent call."""
|
||||
...
|
||||
|
||||
async def transcribe(
|
||||
self,
|
||||
*,
|
||||
@@ -115,3 +130,7 @@ class TranscriptionProvider(Protocol):
|
||||
) -> TranscriptionResult:
|
||||
"""Transcribe the provided image according to the prompt text."""
|
||||
...
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""Release any pooled network resources held by the adapter."""
|
||||
...
|
||||
|
||||
@@ -74,7 +74,10 @@ class _CapturingAsyncClient:
|
||||
try:
|
||||
self.last_body = response.content
|
||||
except httpx.ResponseNotRead:
|
||||
response.stream = _CapturingAsyncByteStream(response.stream, self._capture_body)
|
||||
stream = response.stream
|
||||
if not isinstance(stream, httpx.AsyncByteStream):
|
||||
raise
|
||||
response.stream = _CapturingAsyncByteStream(stream, self._capture_body)
|
||||
return response
|
||||
|
||||
def build_request(self, *args: Any, **kwargs: Any) -> httpx.Request:
|
||||
@@ -195,7 +198,15 @@ class OpenRouterTranscriptionProvider:
|
||||
self._current_request_manifest: RequestManifest | None = None
|
||||
self._current_transport_evidence: TransportEvidence | None = None
|
||||
if client is None:
|
||||
self._capturing_client = _CapturingAsyncClient(async_client or httpx.AsyncClient(follow_redirects=True))
|
||||
# httpx defaults every phase to 5s, which silently caps provider calls far
|
||||
# below worker_provider_timeout_seconds. Track the configured budget instead.
|
||||
timeout = httpx.Timeout(
|
||||
self._settings.worker_provider_timeout_seconds,
|
||||
connect=10.0,
|
||||
)
|
||||
self._capturing_client = _CapturingAsyncClient(
|
||||
async_client or httpx.AsyncClient(follow_redirects=True, timeout=timeout)
|
||||
)
|
||||
client = OpenRouter(
|
||||
api_key=self._settings.openrouter_api_key.get_secret_value(),
|
||||
async_client=self._capturing_client,
|
||||
|
||||
@@ -2,7 +2,12 @@
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import field
|
||||
from typing import Self
|
||||
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from ..config import Settings
|
||||
from .documents import DocumentService
|
||||
from .jobs import JobService
|
||||
from .people import PeopleService
|
||||
@@ -20,3 +25,24 @@ class ServiceBundle:
|
||||
sources: SourceService = field(default_factory=SourceService)
|
||||
jobs: JobService = field(default_factory=JobService)
|
||||
people: PeopleService = field(default_factory=PeopleService)
|
||||
|
||||
@classmethod
|
||||
def from_session_factory(
|
||||
cls,
|
||||
session_factory: async_sessionmaker[AsyncSession] | None = None,
|
||||
*,
|
||||
settings: Settings | None = None,
|
||||
) -> Self:
|
||||
"""Build a bundle whose services all share one session factory and settings."""
|
||||
if session_factory is None:
|
||||
return cls()
|
||||
return cls(
|
||||
documents=DocumentService(session_factory=session_factory, settings=settings),
|
||||
sources=SourceService(session_factory=session_factory, settings=settings),
|
||||
jobs=JobService(session_factory=session_factory, settings=settings),
|
||||
people=PeopleService(session_factory=session_factory, settings=settings),
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""Release provider resources held by the bundle."""
|
||||
await self.sources.aclose()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import asyncio
|
||||
from abc import ABC
|
||||
from collections.abc import Sequence
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
@@ -10,6 +10,8 @@ from ..config import Settings
|
||||
from ..config import get_settings
|
||||
from ..db.session import resolve_session_factory
|
||||
from ..db.session import session_scope
|
||||
from ..errors import AppError
|
||||
from ..errors import ErrorCategory
|
||||
|
||||
|
||||
class ServiceBase(ABC):
|
||||
@@ -17,17 +19,14 @@ class ServiceBase(ABC):
|
||||
|
||||
settings: Settings
|
||||
session_factory: async_sessionmaker[AsyncSession]
|
||||
queue: asyncio.Queue
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: async_sessionmaker[AsyncSession] | None = None,
|
||||
queue: asyncio.Queue | None = None,
|
||||
settings: Settings | None = None,
|
||||
):
|
||||
self.settings = settings or get_settings()
|
||||
self.session_factory = session_factory or resolve_session_factory(settings=self.settings)
|
||||
self.queue = queue or asyncio.Queue()
|
||||
|
||||
@asynccontextmanager
|
||||
async def _session_scope(self, session: AsyncSession | None = None):
|
||||
@@ -58,3 +57,28 @@ class ServiceBase(ABC):
|
||||
|
||||
for obj in refresh:
|
||||
await session.refresh(obj)
|
||||
|
||||
async def _get_or_raise[ModelT](
|
||||
self,
|
||||
model: type[ModelT],
|
||||
entity_id: object,
|
||||
*,
|
||||
session: AsyncSession,
|
||||
error: type[AppError],
|
||||
noun: str,
|
||||
suggestion: str,
|
||||
options: Sequence[Any] = (),
|
||||
) -> ModelT:
|
||||
"""Load an entity by primary key or raise a not-found service error.
|
||||
|
||||
``noun`` and ``suggestion`` are supplied by the caller so each domain
|
||||
keeps its own user-facing wording.
|
||||
"""
|
||||
entity = await session.get(model, entity_id, options=list(options) or None)
|
||||
if entity is None:
|
||||
raise error(
|
||||
f"{noun} with id {entity_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion=suggestion,
|
||||
)
|
||||
return entity
|
||||
|
||||
@@ -2,18 +2,22 @@ import logging
|
||||
import shutil
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC
|
||||
from datetime import date
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import selectinload
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
from sqlmodel import SQLModel
|
||||
from sqlmodel import col
|
||||
from sqlmodel import select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from ..config import Settings
|
||||
from ..db.loading import orm_attribute
|
||||
from ..db.loading import selectinload
|
||||
from ..db.models import Document
|
||||
from ..db.models import DocumentPerson
|
||||
from ..db.models import DocumentType
|
||||
@@ -21,7 +25,9 @@ from ..db.registries import AUTHOR_ROLE_SEMANTIC_KEY
|
||||
from ..errors import AppError
|
||||
from ..errors import ErrorCategory
|
||||
from .base import ServiceBase
|
||||
from .sources import source_mime_type
|
||||
from .registry import RegistryService
|
||||
from .source_media import lookup_source_mime_type
|
||||
from .source_media import supported_source_formats
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -46,19 +52,23 @@ class DocumentTypeError(DocumentError):
|
||||
"""Raised when Document Type maintenance fails."""
|
||||
|
||||
|
||||
def _normalize_registry_label(label: str) -> str:
|
||||
normalized = label.strip()
|
||||
if not normalized:
|
||||
raise DocumentTypeError(
|
||||
"Document Type label is required",
|
||||
category=ErrorCategory.VALIDATION,
|
||||
suggestion="Enter a user-facing label and retry.",
|
||||
)
|
||||
return normalized
|
||||
class DocumentTypeRegistry(RegistryService[DocumentType]):
|
||||
"""Document Type registry maintenance."""
|
||||
|
||||
model = DocumentType
|
||||
error = DocumentTypeError
|
||||
noun = "Document Type"
|
||||
short_noun = "type"
|
||||
referenced_retainer = "historical Documents"
|
||||
|
||||
def _document_type_label_key(label: str) -> str:
|
||||
return _normalize_registry_label(label).casefold()
|
||||
def reference_model(self) -> type[SQLModel]:
|
||||
return Document
|
||||
|
||||
def reference_id_column(self) -> Any:
|
||||
return col(Document.id)
|
||||
|
||||
def reference_key_column(self) -> Any:
|
||||
return col(Document.document_type_id)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -109,6 +119,14 @@ class DocumentPrintProjection:
|
||||
class DocumentService(ServiceBase):
|
||||
"""Thin service class for managing documents in the database."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: async_sessionmaker[AsyncSession] | None = None,
|
||||
settings: Settings | None = None,
|
||||
) -> None:
|
||||
super().__init__(session_factory, settings)
|
||||
self._document_types = DocumentTypeRegistry(self.session_factory, self.settings)
|
||||
|
||||
async def _validate_document_type(self, *, session: AsyncSession, document: Document) -> None:
|
||||
"""Validate the UUID-backed Document Type reference."""
|
||||
if document.document_type_id is None:
|
||||
@@ -120,16 +138,23 @@ class DocumentService(ServiceBase):
|
||||
suggestion="Select a valid document type and retry.",
|
||||
)
|
||||
|
||||
async def _get_document_or_raise(self, *, session: AsyncSession, document_id: UUID) -> Document:
|
||||
"""Get a document by id or raise a not-found service error."""
|
||||
document = await session.get(Document, document_id)
|
||||
if document is None:
|
||||
raise DocumentError(
|
||||
f"Document with id {document_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Verify the document id and retry.",
|
||||
async def _read_document(
|
||||
self,
|
||||
*,
|
||||
session: AsyncSession,
|
||||
document_id: UUID,
|
||||
options: Sequence[Any] = (),
|
||||
suggestion: str = "Verify the document id and retry.",
|
||||
) -> Document:
|
||||
return await self._get_or_raise(
|
||||
Document,
|
||||
document_id,
|
||||
session=session,
|
||||
error=DocumentError,
|
||||
noun="Document",
|
||||
suggestion=suggestion,
|
||||
options=options,
|
||||
)
|
||||
return document
|
||||
|
||||
#
|
||||
# CRUD Operations
|
||||
@@ -161,21 +186,16 @@ class DocumentService(ServiceBase):
|
||||
The selectinload option is used to eagerly load related jobs and sources.
|
||||
"""
|
||||
async with self._session_scope(session) as _session:
|
||||
document = await _session.get(
|
||||
Document,
|
||||
document_id,
|
||||
document = await self._read_document(
|
||||
session=_session,
|
||||
document_id=document_id,
|
||||
options=(
|
||||
selectinload(Document.jobs), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.sources), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.jobs),
|
||||
selectinload(Document.sources),
|
||||
),
|
||||
)
|
||||
if document is None:
|
||||
raise DocumentError(
|
||||
f"Document with id {document_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Re-upload the source document and retry.",
|
||||
)
|
||||
elif not document.sources:
|
||||
if not document.sources:
|
||||
raise MissingSourceError(
|
||||
f"Document with id {document_id} has no associated source records",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
@@ -187,7 +207,6 @@ class DocumentService(ServiceBase):
|
||||
"""Update an existing document in the database."""
|
||||
async with self._session_scope(session) as _session:
|
||||
await self._validate_document_type(session=_session, document=document)
|
||||
document.updated_at = datetime.now(UTC)
|
||||
merged = await _session.merge(document)
|
||||
await self._finalize(session=_session, caller_session=session, refresh=(merged,))
|
||||
return merged
|
||||
@@ -196,21 +215,15 @@ class DocumentService(ServiceBase):
|
||||
"""Delete a document from the database."""
|
||||
document_id = document.id
|
||||
async with self._session_scope(session) as _session:
|
||||
existing = await _session.get(
|
||||
Document,
|
||||
document.id,
|
||||
existing = await self._read_document(
|
||||
session=_session,
|
||||
document_id=document.id,
|
||||
options=(
|
||||
selectinload(Document.jobs), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.sources), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.document_people), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.jobs),
|
||||
selectinload(Document.sources),
|
||||
selectinload(Document.document_people),
|
||||
),
|
||||
)
|
||||
if existing is None:
|
||||
raise DocumentError(
|
||||
f"Document with id {document.id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Verify the document id and retry.",
|
||||
)
|
||||
|
||||
has_jobs = bool(existing.jobs)
|
||||
has_sources = bool(existing.sources)
|
||||
@@ -263,9 +276,9 @@ class DocumentService(ServiceBase):
|
||||
"""List documents with relations needed by the archival table."""
|
||||
async with self._session_scope(session) as _session:
|
||||
query = select(Document).options(
|
||||
selectinload(Document.document_people).selectinload(DocumentPerson.person), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.document_people).selectinload(DocumentPerson.role_ref), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.document_type_ref), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.document_people).selectinload(orm_attribute(DocumentPerson.person)),
|
||||
selectinload(Document.document_people).selectinload(orm_attribute(DocumentPerson.role_ref)),
|
||||
selectinload(Document.document_type_ref),
|
||||
)
|
||||
result = await _session.exec(query)
|
||||
return result.all()
|
||||
@@ -276,11 +289,11 @@ class DocumentService(ServiceBase):
|
||||
query = (
|
||||
select(Document)
|
||||
.options(
|
||||
selectinload(Document.jobs), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.sources), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.document_people).selectinload(DocumentPerson.person), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.document_people).selectinload(DocumentPerson.role_ref), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.document_type_ref), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Document.jobs),
|
||||
selectinload(Document.sources),
|
||||
selectinload(Document.document_people).selectinload(orm_attribute(DocumentPerson.person)),
|
||||
selectinload(Document.document_people).selectinload(orm_attribute(DocumentPerson.role_ref)),
|
||||
selectinload(Document.document_type_ref),
|
||||
)
|
||||
.where(Document.id == document_id)
|
||||
.execution_options(populate_existing=True)
|
||||
@@ -316,7 +329,7 @@ class DocumentService(ServiceBase):
|
||||
DocumentPrintSource(
|
||||
id=source.id,
|
||||
page_number=source.page_number,
|
||||
media_type=source_mime_type(source.filename),
|
||||
media_type=_print_media_type(source.filename),
|
||||
current_text=_current_print_text(source.revised_text, source.raw_transcription),
|
||||
)
|
||||
for source in sorted(document.sources, key=lambda item: (item.page_number, item.id))
|
||||
@@ -354,13 +367,7 @@ class DocumentService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> Sequence[DocumentType]:
|
||||
"""List configured document types."""
|
||||
async with self._session_scope(session) as _session:
|
||||
query = select(DocumentType)
|
||||
if active_only:
|
||||
query = query.where(col(DocumentType.is_active).is_(True))
|
||||
query = query.order_by(col(DocumentType.normalized_label), col(DocumentType.id))
|
||||
result = await _session.exec(query)
|
||||
return result.all()
|
||||
return await self._document_types.list_entries(active_only=active_only, session=session)
|
||||
|
||||
async def list_document_type_summaries(
|
||||
self,
|
||||
@@ -368,21 +375,14 @@ class DocumentService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> Sequence[DocumentTypeSummary]:
|
||||
"""List Document Types alphabetically with current usage counts."""
|
||||
async with self._session_scope(session) as _session:
|
||||
query = (
|
||||
select(DocumentType, func.count(col(Document.id)))
|
||||
.outerjoin(Document, col(Document.document_type_id) == col(DocumentType.id))
|
||||
.group_by(col(DocumentType.id))
|
||||
.order_by(col(DocumentType.normalized_label), col(DocumentType.id))
|
||||
)
|
||||
rows = (await _session.exec(query)).all()
|
||||
rows = await self._document_types.list_entries_with_counts(session=session)
|
||||
return [
|
||||
DocumentTypeSummary(
|
||||
id=document_type.id,
|
||||
label=document_type.label,
|
||||
is_active=document_type.is_active,
|
||||
is_built_in=document_type.semantic_key is not None,
|
||||
document_count=int(document_count),
|
||||
document_count=document_count,
|
||||
)
|
||||
for document_type, document_count in rows
|
||||
]
|
||||
@@ -395,22 +395,7 @@ class DocumentService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> DocumentType:
|
||||
"""Create a UUID-identified Document Type with a unique label."""
|
||||
document_type = DocumentType(
|
||||
label=_normalize_registry_label(label),
|
||||
normalized_label=_document_type_label_key(label),
|
||||
is_active=is_active,
|
||||
)
|
||||
async with self._session_scope(session) as _session:
|
||||
_session.add(document_type)
|
||||
try:
|
||||
await self._finalize(session=_session, caller_session=session, refresh=(document_type,))
|
||||
except IntegrityError as exc:
|
||||
raise DocumentTypeError(
|
||||
f"Document Type label {document_type.label!r} already exists",
|
||||
category=ErrorCategory.CONFLICT,
|
||||
suggestion="Choose a different label or edit the existing type.",
|
||||
) from exc
|
||||
return document_type
|
||||
return await self._document_types.create_entry(label=label, is_active=is_active, session=session)
|
||||
|
||||
async def read_document_type(
|
||||
self,
|
||||
@@ -419,15 +404,7 @@ class DocumentService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> DocumentType:
|
||||
"""Read a Document Type by id."""
|
||||
async with self._session_scope(session) as _session:
|
||||
document_type = await _session.get(DocumentType, document_type_id)
|
||||
if document_type is None:
|
||||
raise DocumentTypeError(
|
||||
f"Document Type with id {document_type_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Refresh Settings and select an available Document Type.",
|
||||
)
|
||||
return document_type
|
||||
return await self._document_types.read_entry(document_type_id, session=session)
|
||||
|
||||
async def update_document_type(
|
||||
self,
|
||||
@@ -438,27 +415,12 @@ class DocumentService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> DocumentType:
|
||||
"""Update a Document Type label and active state."""
|
||||
async with self._session_scope(session) as _session:
|
||||
document_type = await _session.get(DocumentType, document_type_id)
|
||||
if document_type is None:
|
||||
raise DocumentTypeError(
|
||||
f"Document Type with id {document_type_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Refresh Settings and select an available Document Type.",
|
||||
return await self._document_types.update_entry(
|
||||
document_type_id,
|
||||
label=label,
|
||||
is_active=is_active,
|
||||
session=session,
|
||||
)
|
||||
document_type.label = _normalize_registry_label(label)
|
||||
document_type.normalized_label = _document_type_label_key(label)
|
||||
document_type.is_active = is_active
|
||||
document_type.updated_at = datetime.now(UTC)
|
||||
try:
|
||||
await self._finalize(session=_session, caller_session=session, refresh=(document_type,))
|
||||
except IntegrityError as exc:
|
||||
raise DocumentTypeError(
|
||||
f"Document Type label {document_type.label!r} already exists",
|
||||
category=ErrorCategory.CONFLICT,
|
||||
suggestion="Choose a different label or edit the existing type.",
|
||||
) from exc
|
||||
return document_type
|
||||
|
||||
async def delete_document_type(
|
||||
self,
|
||||
@@ -467,28 +429,7 @@ class DocumentService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> None:
|
||||
"""Delete an unreferenced Document Type without cascade behavior."""
|
||||
async with self._session_scope(session) as _session:
|
||||
document_type = await _session.get(DocumentType, document_type_id)
|
||||
if document_type is None:
|
||||
raise DocumentTypeError(
|
||||
f"Document Type with id {document_type_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Refresh Settings and select an available Document Type.",
|
||||
)
|
||||
if document_type.semantic_key is not None:
|
||||
raise DocumentTypeError(
|
||||
f"Built-in Document Type {document_type.label!r} cannot be deleted",
|
||||
category=ErrorCategory.CONFLICT,
|
||||
suggestion="Deactivate the type instead; its built-in meaning must remain available.",
|
||||
)
|
||||
if await self._document_type_is_referenced(session=_session, document_type=document_type):
|
||||
raise DocumentTypeError(
|
||||
f"Document Type {document_type.label!r} is referenced and cannot be deleted",
|
||||
category=ErrorCategory.CONFLICT,
|
||||
suggestion="Deactivate the type instead; historical Documents will retain it.",
|
||||
)
|
||||
await _session.delete(document_type)
|
||||
await self._finalize(session=_session, caller_session=session)
|
||||
await self._document_types.delete_entry(document_type_id, session=session)
|
||||
|
||||
async def is_document_type_referenced(
|
||||
self,
|
||||
@@ -497,29 +438,7 @@ class DocumentService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> bool:
|
||||
"""Return whether a Document references a Document Type."""
|
||||
async with self._session_scope(session) as _session:
|
||||
document_type = await _session.get(DocumentType, document_type_id)
|
||||
if document_type is None:
|
||||
raise DocumentTypeError(
|
||||
f"Document Type with id {document_type_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Refresh Settings and select an available Document Type.",
|
||||
)
|
||||
return await self._document_type_is_referenced(
|
||||
session=_session,
|
||||
document_type=document_type,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _document_type_is_referenced(
|
||||
*,
|
||||
session: AsyncSession,
|
||||
document_type: DocumentType,
|
||||
) -> bool:
|
||||
reference = (
|
||||
await session.exec(select(Document.id).where(Document.document_type_id == document_type.id))
|
||||
).first()
|
||||
return reference is not None
|
||||
return await self._document_types.is_referenced(document_type_id, session=session)
|
||||
|
||||
async def set_document_type(
|
||||
self,
|
||||
@@ -530,14 +449,25 @@ class DocumentService(ServiceBase):
|
||||
) -> Document:
|
||||
"""Set a Document Type by UUID."""
|
||||
async with self._session_scope(session) as _session:
|
||||
document = await self._get_document_or_raise(session=_session, document_id=document_id)
|
||||
document = await self._read_document(session=_session, document_id=document_id)
|
||||
document.document_type_id = document_type_id
|
||||
await self._validate_document_type(session=_session, document=document)
|
||||
document.updated_at = datetime.now(UTC)
|
||||
await self._finalize(session=_session, caller_session=session, refresh=(document,))
|
||||
return document
|
||||
|
||||
|
||||
def _print_media_type(filename: str) -> str:
|
||||
"""Resolve a stored Source filename to its MIME type for print rendering."""
|
||||
mime_type = lookup_source_mime_type(filename)
|
||||
if mime_type is None:
|
||||
raise DocumentError(
|
||||
f"Unsupported Source format: {Path(filename).suffix.lower() or '<none>'}",
|
||||
category=ErrorCategory.USER_INPUT,
|
||||
suggestion=f"Use one of the supported Source formats: {supported_source_formats()}.",
|
||||
)
|
||||
return mime_type
|
||||
|
||||
|
||||
def _current_print_text(revised_text: str | None, raw_transcription: str | None) -> str | None:
|
||||
selected = revised_text if revised_text is not None else raw_transcription
|
||||
return selected if selected is not None and selected.strip() else None
|
||||
|
||||
@@ -6,10 +6,12 @@ from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import selectinload
|
||||
from sqlmodel import col
|
||||
from sqlmodel import select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from ..db.loading import orm_attribute
|
||||
from ..db.loading import selectinload
|
||||
from ..db.models import ExecutionAttempt
|
||||
from ..db.models import Job
|
||||
from ..db.models import JobSource
|
||||
@@ -64,8 +66,8 @@ class JobService(ServiceBase):
|
||||
query = (
|
||||
select(Job)
|
||||
.options(
|
||||
selectinload(Job.document), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Job.job_sources).selectinload(JobSource.source), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Job.document),
|
||||
selectinload(Job.job_sources).selectinload(orm_attribute(JobSource.source)),
|
||||
)
|
||||
.where(Job.id == job_id)
|
||||
.execution_options(populate_existing=True)
|
||||
@@ -100,28 +102,28 @@ class JobService(ServiceBase):
|
||||
"""Query jobs from the database based on provided filters."""
|
||||
async with self._session_scope(session) as _session:
|
||||
query = select(Job).options(
|
||||
selectinload(Job.document), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Job.job_sources).selectinload(JobSource.source), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Job.document),
|
||||
selectinload(Job.job_sources).selectinload(orm_attribute(JobSource.source)),
|
||||
)
|
||||
if status is not None:
|
||||
query = query.where(Job.status == status)
|
||||
if filename is not None:
|
||||
query = query.where(Job.job_sources.any(JobSource.source.has(Source.filename == filename)))
|
||||
query = query.where(
|
||||
col(Job.job_sources).any(col(JobSource.source).has(col(Source.filename) == filename))
|
||||
)
|
||||
result = await _session.exec(query)
|
||||
return result.all()
|
||||
|
||||
async def list_jobs(
|
||||
self,
|
||||
*,
|
||||
load_docs: bool = False,
|
||||
session: AsyncSession | None = None,
|
||||
) -> Sequence[Job]:
|
||||
"""List all jobs in the database with eagerly loaded documents."""
|
||||
_ = load_docs
|
||||
async with self._session_scope(session) as _session:
|
||||
query = select(Job).options(
|
||||
selectinload(Job.document), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Job.job_sources).selectinload(JobSource.source), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Job.document),
|
||||
selectinload(Job.job_sources).selectinload(orm_attribute(JobSource.source)),
|
||||
)
|
||||
result = await _session.exec(query)
|
||||
return result.all()
|
||||
@@ -153,7 +155,10 @@ class JobService(ServiceBase):
|
||||
async with self._session_scope(session) as _session:
|
||||
query = (
|
||||
select(Job)
|
||||
.options(selectinload(Job.document)) # pyright: ignore[reportArgumentType]
|
||||
.options(
|
||||
selectinload(Job.document),
|
||||
selectinload(Job.job_sources).selectinload(orm_attribute(JobSource.source)),
|
||||
)
|
||||
.where(Job.id == job_id)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
@@ -163,28 +168,39 @@ class JobService(ServiceBase):
|
||||
job.status = status
|
||||
if retry_count_increment:
|
||||
job.retry_count += retry_count_increment
|
||||
job.date_updated = datetime.now(UTC)
|
||||
await self._finalize(session=_session, caller_session=session, refresh=(job,))
|
||||
return job
|
||||
|
||||
async def read_next_queued_job(
|
||||
async def claim_next_queued_job(
|
||||
self,
|
||||
*,
|
||||
session: AsyncSession | None = None,
|
||||
) -> Job | None:
|
||||
"""Read the next queued job ordered by creation time."""
|
||||
"""Atomically claim the oldest queued job by transitioning it to PROCESSING.
|
||||
|
||||
The selection is deliberately unadorned: no eager loads are applied to the
|
||||
hot poll, because callers re-read the claimed job with the relationships
|
||||
they actually need. On PostgreSQL the row is locked with ``SKIP LOCKED`` so
|
||||
concurrent workers never contend for the same job.
|
||||
"""
|
||||
async with self._session_scope(session) as _session:
|
||||
query = (
|
||||
select(Job)
|
||||
.options(
|
||||
selectinload(Job.document), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Job.job_sources).selectinload(JobSource.source), # pyright: ignore[reportArgumentType]
|
||||
)
|
||||
.where(Job.status == JobStatus.QUEUED)
|
||||
# Break ties by id so "next" is stable when two rows share close timestamps.
|
||||
.order_by(Job.date_created, Job.id) # pyright: ignore[reportArgumentType]
|
||||
.order_by(col(Job.date_created), col(Job.id))
|
||||
.limit(1)
|
||||
)
|
||||
return (await _session.exec(query)).first()
|
||||
if _session.get_bind().dialect.name == "postgresql":
|
||||
query = query.with_for_update(skip_locked=True)
|
||||
|
||||
job = (await _session.exec(query)).first()
|
||||
if job is None:
|
||||
return None
|
||||
|
||||
job.status = JobStatus.PROCESSING
|
||||
await self._finalize(session=_session, caller_session=session, refresh=(job,))
|
||||
return job
|
||||
|
||||
async def requeue_stale_processing_jobs(
|
||||
self,
|
||||
@@ -292,7 +308,7 @@ class JobService(ServiceBase):
|
||||
(
|
||||
await session.exec(
|
||||
select(ProcessingArtifact).where(
|
||||
ProcessingArtifact.execution_attempt_id.in_(attempt_ids)
|
||||
col(ProcessingArtifact.execution_attempt_id).in_(attempt_ids)
|
||||
)
|
||||
)
|
||||
).all()
|
||||
@@ -339,7 +355,7 @@ class JobService(ServiceBase):
|
||||
query = (
|
||||
select(Job)
|
||||
.options(
|
||||
selectinload(Job.job_sources).selectinload(JobSource.source), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Job.job_sources).selectinload(orm_attribute(JobSource.source)),
|
||||
)
|
||||
.where(Job.id == job_id)
|
||||
.execution_options(populate_existing=True)
|
||||
@@ -376,7 +392,7 @@ class JobService(ServiceBase):
|
||||
query = (
|
||||
select(Job)
|
||||
.options(
|
||||
selectinload(Job.job_sources).selectinload(JobSource.source), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Job.job_sources).selectinload(orm_attribute(JobSource.source)),
|
||||
)
|
||||
.where(Job.id == job_id)
|
||||
.execution_options(populate_existing=True)
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Single implementation for persisting uploaded media bytes to disk.
|
||||
|
||||
Source pages, Person portraits, and homepage images all follow the same
|
||||
sequence: resolve a target directory, create it, write the bytes, and translate
|
||||
an ``OSError`` into a domain error. The write itself runs on a worker thread so
|
||||
it never blocks the event loop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from ..errors import AppError
|
||||
from ..errors import ErrorCategory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def build_stored_filename(*, filename: str, filename_stem: str | None = None) -> str:
|
||||
"""Return a safe stored filename preserving the submitted extension."""
|
||||
safe_name = Path(filename).name
|
||||
suffix = Path(safe_name).suffix.lower()
|
||||
stem = filename_stem or str(uuid4())
|
||||
return f"{stem}{suffix}"
|
||||
|
||||
|
||||
async def write_media_bytes(
|
||||
*,
|
||||
target_dir: Path,
|
||||
stored_name: str,
|
||||
file_bytes: bytes,
|
||||
error: type[AppError],
|
||||
failure_message: str,
|
||||
failure_suggestion: str,
|
||||
log_label: str,
|
||||
) -> Path:
|
||||
"""Create ``target_dir`` and write ``file_bytes`` into it off the event loop."""
|
||||
stored_path = target_dir / stored_name
|
||||
try:
|
||||
await asyncio.to_thread(_write, stored_path, file_bytes)
|
||||
except OSError as exc:
|
||||
raise error(
|
||||
failure_message,
|
||||
category=ErrorCategory.INFRA_PERSISTENT,
|
||||
suggestion=failure_suggestion,
|
||||
) from exc
|
||||
|
||||
logger.info("Stored %s: %s", log_label, stored_path)
|
||||
return stored_path
|
||||
|
||||
|
||||
def _write(stored_path: Path, file_bytes: bytes) -> None:
|
||||
stored_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
stored_path.write_bytes(file_bytes)
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
from dataclasses import dataclass
|
||||
@@ -9,6 +10,7 @@ from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
from PIL import UnidentifiedImageError
|
||||
from PIL.TiffImagePlugin import TiffImageFile
|
||||
|
||||
from transcription.errors import AppError
|
||||
from transcription.errors import ErrorCategory
|
||||
@@ -43,14 +45,16 @@ class OrientationNormalization:
|
||||
original_height: int
|
||||
derivative_width: int
|
||||
derivative_height: int
|
||||
|
||||
@property
|
||||
def digest_sha256(self) -> str:
|
||||
return hashlib.sha256(self.content).hexdigest()
|
||||
# Computed eagerly by `normalize_orientation`, which already runs off the
|
||||
# event loop, so callers never hash multi-megabyte derivatives inline.
|
||||
digest_sha256: str
|
||||
|
||||
|
||||
def normalize_orientation(path: str | Path, *, media_type: str) -> OrientationNormalization | None:
|
||||
"""Physically apply supported EXIF rotation, returning None for a safe no-op."""
|
||||
"""Physically apply supported EXIF rotation, returning None for a safe no-op.
|
||||
|
||||
Blocking. Async callers must use :func:`normalize_orientation_async`.
|
||||
"""
|
||||
source_path = Path(path)
|
||||
if media_type not in {"image/jpeg", "image/png", "image/tiff"}:
|
||||
return None
|
||||
@@ -62,7 +66,7 @@ def normalize_orientation(path: str | Path, *, media_type: str) -> OrientationNo
|
||||
return None
|
||||
|
||||
transpose, rotation = transformation
|
||||
if image.format == "TIFF":
|
||||
if isinstance(image, TiffImageFile):
|
||||
original_width = int(image.tag_v2.get(256, image.width))
|
||||
original_height = int(image.tag_v2.get(257, image.height))
|
||||
# Pillow applies TIFF orientation while decoding; copying freezes those upright pixels.
|
||||
@@ -88,8 +92,9 @@ def normalize_orientation(path: str | Path, *, media_type: str) -> OrientationNo
|
||||
) from exc
|
||||
|
||||
suffix = source_path.suffix.lower()
|
||||
content = output.getvalue()
|
||||
return OrientationNormalization(
|
||||
content=output.getvalue(),
|
||||
content=content,
|
||||
media_type=media_type,
|
||||
suffix=suffix,
|
||||
original_orientation=orientation,
|
||||
@@ -98,4 +103,15 @@ def normalize_orientation(path: str | Path, *, media_type: str) -> OrientationNo
|
||||
original_height=original_height,
|
||||
derivative_width=normalized.width,
|
||||
derivative_height=normalized.height,
|
||||
digest_sha256=hashlib.sha256(content).hexdigest(),
|
||||
)
|
||||
|
||||
|
||||
async def normalize_orientation_async(path: str | Path, *, media_type: str) -> OrientationNormalization | None:
|
||||
"""Run :func:`normalize_orientation` off the event loop.
|
||||
|
||||
Pillow decode, transpose, and re-encode are CPU- and disk-bound and scale
|
||||
with page size, so they must not run on the request or worker event loop
|
||||
([MED-01]).
|
||||
"""
|
||||
return await asyncio.to_thread(normalize_orientation, path, media_type=media_type)
|
||||
|
||||
@@ -6,20 +6,21 @@ import logging
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import selectinload
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
from sqlmodel import SQLModel
|
||||
from sqlmodel import col
|
||||
from sqlmodel import select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from ..config import Settings
|
||||
from ..config import get_settings
|
||||
from ..db.loading import orm_attribute
|
||||
from ..db.loading import selectinload
|
||||
from ..db.models import Document
|
||||
from ..db.models import DocumentPerson
|
||||
from ..db.models import Person
|
||||
@@ -27,6 +28,9 @@ from ..db.models import PersonRole
|
||||
from ..errors import AppError
|
||||
from ..errors import ErrorCategory
|
||||
from .base import ServiceBase
|
||||
from .media_storage import build_stored_filename
|
||||
from .media_storage import write_media_bytes
|
||||
from .registry import RegistryService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -46,15 +50,23 @@ class PersonRoleError(PeopleError):
|
||||
"""Raised when Person Role maintenance fails."""
|
||||
|
||||
|
||||
def _normalize_role_label(label: str) -> str:
|
||||
normalized = label.strip()
|
||||
if not normalized:
|
||||
raise PersonRoleError(
|
||||
"Person Role label is required",
|
||||
category=ErrorCategory.VALIDATION,
|
||||
suggestion="Enter a user-facing label and retry.",
|
||||
)
|
||||
return normalized
|
||||
class PersonRoleRegistry(RegistryService[PersonRole]):
|
||||
"""Person Role registry maintenance."""
|
||||
|
||||
model = PersonRole
|
||||
error = PersonRoleError
|
||||
noun = "Person Role"
|
||||
short_noun = "role"
|
||||
referenced_retainer = "historical relationships"
|
||||
|
||||
def reference_model(self) -> type[SQLModel]:
|
||||
return DocumentPerson
|
||||
|
||||
def reference_id_column(self) -> Any:
|
||||
return col(DocumentPerson.id)
|
||||
|
||||
def reference_key_column(self) -> Any:
|
||||
return col(DocumentPerson.role_id)
|
||||
|
||||
|
||||
def normalize_family_search_id(value: str | None) -> str | None:
|
||||
@@ -71,10 +83,6 @@ def normalize_family_search_id(value: str | None) -> str | None:
|
||||
return normalized
|
||||
|
||||
|
||||
def _person_role_label_key(label: str) -> str:
|
||||
return _normalize_role_label(label).casefold()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PersonRoleSummary:
|
||||
"""Settings read model for a Person Role and its usage count."""
|
||||
@@ -97,6 +105,14 @@ class DocumentPersonInput:
|
||||
class PeopleService(ServiceBase):
|
||||
"""Manage People, relationship roles, and document-person links."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: async_sessionmaker[AsyncSession] | None = None,
|
||||
settings: Settings | None = None,
|
||||
) -> None:
|
||||
super().__init__(session_factory, settings)
|
||||
self._person_roles = PersonRoleRegistry(self.session_factory, self.settings)
|
||||
|
||||
async def create_person(self, person: Person, *, session: AsyncSession | None = None) -> Person:
|
||||
async with self._session_scope(session) as _session:
|
||||
person.family_search_id = normalize_family_search_id(person.family_search_id)
|
||||
@@ -117,7 +133,6 @@ class PeopleService(ServiceBase):
|
||||
async def update_person(self, person: Person, *, session: AsyncSession | None = None) -> Person:
|
||||
async with self._session_scope(session) as _session:
|
||||
person.family_search_id = normalize_family_search_id(person.family_search_id)
|
||||
person.updated_at = datetime.now(UTC)
|
||||
merged = await _session.merge(person)
|
||||
try:
|
||||
await self._finalize(session=_session, caller_session=session, refresh=(merged,))
|
||||
@@ -130,7 +145,7 @@ class PeopleService(ServiceBase):
|
||||
existing = await _session.get(
|
||||
Person,
|
||||
person.id,
|
||||
options=(selectinload(Person.document_people),), # pyright: ignore[reportArgumentType]
|
||||
options=(selectinload(Person.document_people),),
|
||||
)
|
||||
if existing is None:
|
||||
raise self._not_found(f"Person with id {person.id} not found")
|
||||
@@ -177,7 +192,6 @@ class PeopleService(ServiceBase):
|
||||
role_id=document_person.role_id,
|
||||
require_active=existing.role_id != document_person.role_id,
|
||||
)
|
||||
document_person.updated_at = datetime.now(UTC)
|
||||
merged = await _session.merge(document_person)
|
||||
return await self._finalize_link(session=_session, caller_session=session, link=merged)
|
||||
|
||||
@@ -196,8 +210,8 @@ class PeopleService(ServiceBase):
|
||||
query = (
|
||||
select(Person)
|
||||
.options(
|
||||
selectinload(Person.document_people).selectinload(DocumentPerson.document), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Person.document_people).selectinload(DocumentPerson.role_ref), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Person.document_people).selectinload(orm_attribute(DocumentPerson.document)),
|
||||
selectinload(Person.document_people).selectinload(orm_attribute(DocumentPerson.role_ref)),
|
||||
)
|
||||
.where(Person.id == person_id)
|
||||
.execution_options(populate_existing=True)
|
||||
@@ -217,11 +231,7 @@ class PeopleService(ServiceBase):
|
||||
active_only: bool = True,
|
||||
session: AsyncSession | None = None,
|
||||
) -> Sequence[PersonRole]:
|
||||
async with self._session_scope(session) as _session:
|
||||
query = select(PersonRole)
|
||||
if active_only:
|
||||
query = query.where(PersonRole.is_active.is_(True))
|
||||
return (await _session.exec(query.order_by(PersonRole.normalized_label, PersonRole.id))).all()
|
||||
return await self._person_roles.list_entries(active_only=active_only, session=session)
|
||||
|
||||
async def list_person_role_summaries(
|
||||
self,
|
||||
@@ -229,21 +239,14 @@ class PeopleService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> Sequence[PersonRoleSummary]:
|
||||
"""List Person Roles alphabetically with current link counts."""
|
||||
async with self._session_scope(session) as _session:
|
||||
query = (
|
||||
select(PersonRole, func.count(DocumentPerson.id))
|
||||
.outerjoin(DocumentPerson, DocumentPerson.role_id == PersonRole.id)
|
||||
.group_by(PersonRole.id)
|
||||
.order_by(PersonRole.normalized_label, PersonRole.id)
|
||||
)
|
||||
rows = (await _session.exec(query)).all()
|
||||
rows = await self._person_roles.list_entries_with_counts(session=session)
|
||||
return [
|
||||
PersonRoleSummary(
|
||||
id=role.id,
|
||||
label=role.label,
|
||||
is_active=role.is_active,
|
||||
is_built_in=role.semantic_key is not None,
|
||||
link_count=int(link_count),
|
||||
link_count=link_count,
|
||||
)
|
||||
for role, link_count in rows
|
||||
]
|
||||
@@ -256,22 +259,7 @@ class PeopleService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> PersonRole:
|
||||
"""Create a custom Person Role with a unique label."""
|
||||
role = PersonRole(
|
||||
label=_normalize_role_label(label),
|
||||
normalized_label=_person_role_label_key(label),
|
||||
is_active=is_active,
|
||||
)
|
||||
async with self._session_scope(session) as _session:
|
||||
_session.add(role)
|
||||
try:
|
||||
await self._finalize(session=_session, caller_session=session, refresh=(role,))
|
||||
except IntegrityError as exc:
|
||||
raise PersonRoleError(
|
||||
f"Person Role label {role.label!r} already exists",
|
||||
category=ErrorCategory.CONFLICT,
|
||||
suggestion="Choose a different label or edit the existing role.",
|
||||
) from exc
|
||||
return role
|
||||
return await self._person_roles.create_entry(label=label, is_active=is_active, session=session)
|
||||
|
||||
async def read_person_role(
|
||||
self,
|
||||
@@ -280,15 +268,7 @@ class PeopleService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> PersonRole:
|
||||
"""Read a Person Role by id."""
|
||||
async with self._session_scope(session) as _session:
|
||||
role = await _session.get(PersonRole, person_role_id)
|
||||
if role is None:
|
||||
raise PersonRoleError(
|
||||
f"Person Role with id {person_role_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Refresh Settings and select an available Person Role.",
|
||||
)
|
||||
return role
|
||||
return await self._person_roles.read_entry(person_role_id, session=session)
|
||||
|
||||
async def update_person_role(
|
||||
self,
|
||||
@@ -299,27 +279,12 @@ class PeopleService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> PersonRole:
|
||||
"""Update mutable Person Role fields without changing semantic identity."""
|
||||
async with self._session_scope(session) as _session:
|
||||
role = await _session.get(PersonRole, person_role_id)
|
||||
if role is None:
|
||||
raise PersonRoleError(
|
||||
f"Person Role with id {person_role_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Refresh Settings and select an available Person Role.",
|
||||
return await self._person_roles.update_entry(
|
||||
person_role_id,
|
||||
label=label,
|
||||
is_active=is_active,
|
||||
session=session,
|
||||
)
|
||||
role.label = _normalize_role_label(label)
|
||||
role.normalized_label = _person_role_label_key(label)
|
||||
role.is_active = is_active
|
||||
role.updated_at = datetime.now(UTC)
|
||||
try:
|
||||
await self._finalize(session=_session, caller_session=session, refresh=(role,))
|
||||
except IntegrityError as exc:
|
||||
raise PersonRoleError(
|
||||
f"Person Role label {role.label!r} already exists",
|
||||
category=ErrorCategory.CONFLICT,
|
||||
suggestion="Choose a different label or edit the existing role.",
|
||||
) from exc
|
||||
return role
|
||||
|
||||
async def delete_person_role(
|
||||
self,
|
||||
@@ -328,28 +293,7 @@ class PeopleService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> None:
|
||||
"""Delete an unreferenced Person Role without cascade behavior."""
|
||||
async with self._session_scope(session) as _session:
|
||||
role = await _session.get(PersonRole, person_role_id)
|
||||
if role is None:
|
||||
raise PersonRoleError(
|
||||
f"Person Role with id {person_role_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Refresh Settings and select an available Person Role.",
|
||||
)
|
||||
if role.semantic_key is not None:
|
||||
raise PersonRoleError(
|
||||
f"Built-in Person Role {role.label!r} cannot be deleted",
|
||||
category=ErrorCategory.CONFLICT,
|
||||
suggestion="Deactivate the role instead; its built-in meaning must remain available.",
|
||||
)
|
||||
if await self._person_role_is_referenced(session=_session, role=role):
|
||||
raise PersonRoleError(
|
||||
f"Person Role {role.label!r} is referenced and cannot be deleted",
|
||||
category=ErrorCategory.CONFLICT,
|
||||
suggestion="Deactivate the role instead; historical relationships will retain it.",
|
||||
)
|
||||
await _session.delete(role)
|
||||
await self._finalize(session=_session, caller_session=session)
|
||||
await self._person_roles.delete_entry(person_role_id, session=session)
|
||||
|
||||
async def is_person_role_referenced(
|
||||
self,
|
||||
@@ -358,24 +302,7 @@ class PeopleService(ServiceBase):
|
||||
session: AsyncSession | None = None,
|
||||
) -> bool:
|
||||
"""Return whether a document-person link references a Person Role."""
|
||||
async with self._session_scope(session) as _session:
|
||||
role = await _session.get(PersonRole, person_role_id)
|
||||
if role is None:
|
||||
raise PersonRoleError(
|
||||
f"Person Role with id {person_role_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Refresh Settings and select an available Person Role.",
|
||||
)
|
||||
return await self._person_role_is_referenced(session=_session, role=role)
|
||||
|
||||
@staticmethod
|
||||
async def _person_role_is_referenced(
|
||||
*,
|
||||
session: AsyncSession,
|
||||
role: PersonRole,
|
||||
) -> bool:
|
||||
reference = (await session.exec(select(DocumentPerson.id).where(DocumentPerson.role_id == role.id))).first()
|
||||
return reference is not None
|
||||
return await self._person_roles.is_referenced(person_role_id, session=session)
|
||||
|
||||
async def read_person_role_by_semantic_key(
|
||||
self,
|
||||
@@ -403,9 +330,9 @@ class PeopleService(ServiceBase):
|
||||
) -> Sequence[DocumentPerson]:
|
||||
async with self._session_scope(session) as _session:
|
||||
query = select(DocumentPerson).options(
|
||||
selectinload(DocumentPerson.document), # pyright: ignore[reportArgumentType]
|
||||
selectinload(DocumentPerson.person), # pyright: ignore[reportArgumentType]
|
||||
selectinload(DocumentPerson.role_ref), # pyright: ignore[reportArgumentType]
|
||||
selectinload(DocumentPerson.document),
|
||||
selectinload(DocumentPerson.person),
|
||||
selectinload(DocumentPerson.role_ref),
|
||||
)
|
||||
if document_id is not None:
|
||||
query = query.where(DocumentPerson.document_id == document_id)
|
||||
@@ -448,7 +375,6 @@ class PeopleService(ServiceBase):
|
||||
require_active=link.role_id != role_id,
|
||||
)
|
||||
link.role_id = role_id
|
||||
link.updated_at = datetime.now(UTC)
|
||||
return await self._finalize_link(session=_session, caller_session=session, link=link)
|
||||
|
||||
async def remove_document_person_link(
|
||||
@@ -520,7 +446,6 @@ class PeopleService(ServiceBase):
|
||||
_session.add(existing)
|
||||
elif existing.role_id != desired.role_id:
|
||||
existing.role_id = desired.role_id
|
||||
existing.updated_at = datetime.now(UTC)
|
||||
synchronized.append(existing)
|
||||
|
||||
try:
|
||||
@@ -570,6 +495,8 @@ class PeopleService(ServiceBase):
|
||||
category=ErrorCategory.CONFLICT,
|
||||
suggestion="Edit the existing relationship instead of adding another one.",
|
||||
) from exc
|
||||
# Relationships load explicitly; the models declare lazy="raise".
|
||||
await session.refresh(link, attribute_names=["document", "person", "role_ref"])
|
||||
return link
|
||||
|
||||
async def _require_document(self, *, session: AsyncSession, document_id: UUID) -> None:
|
||||
@@ -593,7 +520,7 @@ class PeopleService(ServiceBase):
|
||||
)
|
||||
|
||||
|
||||
def store_person_portrait(
|
||||
async def store_person_portrait(
|
||||
*,
|
||||
person_id: UUID,
|
||||
filename: str,
|
||||
@@ -616,16 +543,12 @@ def store_person_portrait(
|
||||
)
|
||||
|
||||
runtime_settings = settings or get_settings()
|
||||
target_dir = runtime_settings.upload_dir / "persons" / str(person_id)
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
stored_path = target_dir / f"{uuid4()}{suffix}"
|
||||
try:
|
||||
stored_path.write_bytes(file_bytes)
|
||||
except OSError as exc:
|
||||
raise PersonMediaError(
|
||||
"Failed to persist Person portrait media",
|
||||
category=ErrorCategory.INFRA_PERSISTENT,
|
||||
suggestion="Check media directory permissions and available disk space, then retry.",
|
||||
) from exc
|
||||
logger.info("Stored Person portrait media: %s", stored_path)
|
||||
return stored_path
|
||||
return await write_media_bytes(
|
||||
target_dir=runtime_settings.upload_dir / "persons" / str(person_id),
|
||||
stored_name=build_stored_filename(filename=filename),
|
||||
file_bytes=file_bytes,
|
||||
error=PersonMediaError,
|
||||
failure_message="Failed to persist Person portrait media",
|
||||
failure_suggestion="Check media directory permissions and available disk space, then retry.",
|
||||
log_label="Person portrait media",
|
||||
)
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Shared implementation for label-keyed registry tables.
|
||||
|
||||
Document Types and Person Roles are the same shape: a UUID-identified row with a
|
||||
user-facing ``label``, a casefolded ``normalized_label`` uniqueness key, an
|
||||
``is_active`` flag, and an optional ``semantic_key`` marking built-in entries
|
||||
that may be deactivated but never deleted. This module owns that behavior once
|
||||
so the two registries cannot drift apart.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
from typing import Protocol
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlmodel import SQLModel
|
||||
from sqlmodel import col
|
||||
from sqlmodel import select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from ..errors import AppError
|
||||
from ..errors import ErrorCategory
|
||||
from .base import ServiceBase
|
||||
|
||||
|
||||
class RegistryEntry(Protocol):
|
||||
"""Structural contract every registry table row satisfies.
|
||||
|
||||
Bounding ``RegistryService`` by this protocol rather than by bare ``SQLModel``
|
||||
lets the shared implementation read ``id``/``label``/``normalized_label``/
|
||||
``is_active`` off the model class without suppressions.
|
||||
"""
|
||||
|
||||
id: UUID
|
||||
label: str
|
||||
normalized_label: str
|
||||
is_active: bool
|
||||
|
||||
def __init__(self, /, **data: Any) -> None: ...
|
||||
|
||||
|
||||
class RegistryService[ModelT: RegistryEntry](ServiceBase):
|
||||
"""Generic create/read/update/delete behavior for a registry table.
|
||||
|
||||
Subclasses declare the model, the error type, the user-facing noun, and the
|
||||
reference query used to decide whether an entry may be deleted.
|
||||
"""
|
||||
|
||||
#: Registry table this service maintains.
|
||||
model: type[ModelT]
|
||||
#: Error raised for every failure mode of this registry.
|
||||
error: type[AppError]
|
||||
#: User-facing singular noun, e.g. ``"Document Type"``.
|
||||
noun: str
|
||||
#: Lowercase noun used inside remediation suggestions, e.g. ``"type"``.
|
||||
short_noun: str
|
||||
#: Subject that retains a referenced entry, e.g. ``"historical Documents"``.
|
||||
referenced_retainer: str
|
||||
|
||||
@abstractmethod
|
||||
def reference_model(self) -> type[SQLModel]:
|
||||
"""Return the table whose rows reference this registry."""
|
||||
|
||||
@abstractmethod
|
||||
def reference_id_column(self) -> Any:
|
||||
"""Return the primary key column of the referencing table."""
|
||||
|
||||
@abstractmethod
|
||||
def reference_key_column(self) -> Any:
|
||||
"""Return the foreign key column pointing at this registry.
|
||||
|
||||
Declared as methods rather than class attributes because a mapped
|
||||
column stored on a plain class would be re-invoked as a descriptor.
|
||||
"""
|
||||
|
||||
#
|
||||
# Message templates
|
||||
#
|
||||
|
||||
def _not_found(self, entry_id: UUID) -> AppError:
|
||||
return self.error(
|
||||
f"{self.noun} with id {entry_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion=f"Refresh Settings and select an available {self.noun}.",
|
||||
)
|
||||
|
||||
def _duplicate_label(self, label: str) -> AppError:
|
||||
return self.error(
|
||||
f"{self.noun} label {label!r} already exists",
|
||||
category=ErrorCategory.CONFLICT,
|
||||
suggestion=f"Choose a different label or edit the existing {self.short_noun}.",
|
||||
)
|
||||
|
||||
def normalize_label(self, label: str) -> str:
|
||||
"""Strip a submitted label, rejecting blank input."""
|
||||
normalized = label.strip()
|
||||
if not normalized:
|
||||
raise self.error(
|
||||
f"{self.noun} label is required",
|
||||
category=ErrorCategory.VALIDATION,
|
||||
suggestion="Enter a user-facing label and retry.",
|
||||
)
|
||||
return normalized
|
||||
|
||||
def label_key(self, label: str) -> str:
|
||||
"""Return the casefolded uniqueness key for a submitted label."""
|
||||
return self.normalize_label(label).casefold()
|
||||
|
||||
#
|
||||
# Reads
|
||||
#
|
||||
|
||||
async def list_entries(
|
||||
self,
|
||||
*,
|
||||
active_only: bool = True,
|
||||
session: AsyncSession | None = None,
|
||||
) -> Sequence[ModelT]:
|
||||
"""List registry entries alphabetically by normalized label."""
|
||||
async with self._session_scope(session) as _session:
|
||||
query = select(self.model)
|
||||
if active_only:
|
||||
query = query.where(col(self.model.is_active).is_(True))
|
||||
query = query.order_by(col(self.model.normalized_label), col(self.model.id))
|
||||
return (await _session.exec(query)).all()
|
||||
|
||||
async def list_entries_with_counts(
|
||||
self,
|
||||
*,
|
||||
session: AsyncSession | None = None,
|
||||
) -> Sequence[tuple[ModelT, int]]:
|
||||
"""List every entry alphabetically with its current reference count."""
|
||||
async with self._session_scope(session) as _session:
|
||||
query = (
|
||||
select(self.model, func.count(self.reference_id_column()))
|
||||
.outerjoin(self.reference_model(), self.reference_key_column() == col(self.model.id))
|
||||
.group_by(col(self.model.id))
|
||||
.order_by(col(self.model.normalized_label), col(self.model.id))
|
||||
)
|
||||
rows = (await _session.exec(query)).all()
|
||||
return [(entry, int(count)) for entry, count in rows]
|
||||
|
||||
async def read_entry(
|
||||
self,
|
||||
entry_id: UUID,
|
||||
*,
|
||||
session: AsyncSession | None = None,
|
||||
) -> ModelT:
|
||||
"""Read a registry entry by id."""
|
||||
async with self._session_scope(session) as _session:
|
||||
entry = await _session.get(self.model, entry_id)
|
||||
if entry is None:
|
||||
raise self._not_found(entry_id)
|
||||
return entry
|
||||
|
||||
async def is_referenced(
|
||||
self,
|
||||
entry_id: UUID,
|
||||
*,
|
||||
session: AsyncSession | None = None,
|
||||
) -> bool:
|
||||
"""Return whether any row references the registry entry."""
|
||||
async with self._session_scope(session) as _session:
|
||||
entry = await _session.get(self.model, entry_id)
|
||||
if entry is None:
|
||||
raise self._not_found(entry_id)
|
||||
return await self._is_referenced(session=_session, entry=entry)
|
||||
|
||||
async def _is_referenced(self, *, session: AsyncSession, entry: ModelT) -> bool:
|
||||
query = select(self.reference_id_column()).where(self.reference_key_column() == entry.id)
|
||||
return (await session.exec(query)).first() is not None
|
||||
|
||||
#
|
||||
# Writes
|
||||
#
|
||||
|
||||
async def create_entry(
|
||||
self,
|
||||
*,
|
||||
label: str,
|
||||
is_active: bool = True,
|
||||
session: AsyncSession | None = None,
|
||||
) -> ModelT:
|
||||
"""Create a UUID-identified entry with a unique label."""
|
||||
entry = self.model(
|
||||
label=self.normalize_label(label),
|
||||
normalized_label=self.label_key(label),
|
||||
is_active=is_active,
|
||||
)
|
||||
async with self._session_scope(session) as _session:
|
||||
_session.add(entry)
|
||||
try:
|
||||
await self._finalize(session=_session, caller_session=session, refresh=(entry,))
|
||||
except IntegrityError as exc:
|
||||
raise self._duplicate_label(entry.label) from exc
|
||||
return entry
|
||||
|
||||
async def update_entry(
|
||||
self,
|
||||
entry_id: UUID,
|
||||
*,
|
||||
label: str,
|
||||
is_active: bool,
|
||||
session: AsyncSession | None = None,
|
||||
) -> ModelT:
|
||||
"""Update mutable fields without changing semantic identity."""
|
||||
async with self._session_scope(session) as _session:
|
||||
entry = await _session.get(self.model, entry_id)
|
||||
if entry is None:
|
||||
raise self._not_found(entry_id)
|
||||
entry.label = self.normalize_label(label)
|
||||
entry.normalized_label = self.label_key(label)
|
||||
entry.is_active = is_active
|
||||
try:
|
||||
await self._finalize(session=_session, caller_session=session, refresh=(entry,))
|
||||
except IntegrityError as exc:
|
||||
raise self._duplicate_label(entry.label) from exc
|
||||
return entry
|
||||
|
||||
async def delete_entry(
|
||||
self,
|
||||
entry_id: UUID,
|
||||
*,
|
||||
session: AsyncSession | None = None,
|
||||
) -> None:
|
||||
"""Delete an unreferenced, non-built-in entry without cascade behavior."""
|
||||
async with self._session_scope(session) as _session:
|
||||
entry = await _session.get(self.model, entry_id)
|
||||
if entry is None:
|
||||
raise self._not_found(entry_id)
|
||||
if entry.semantic_key is not None:
|
||||
raise self.error(
|
||||
f"Built-in {self.noun} {entry.label!r} cannot be deleted",
|
||||
category=ErrorCategory.CONFLICT,
|
||||
suggestion=(
|
||||
f"Deactivate the {self.short_noun} instead; "
|
||||
"its built-in meaning must remain available."
|
||||
),
|
||||
)
|
||||
if await self._is_referenced(session=_session, entry=entry):
|
||||
raise self.error(
|
||||
f"{self.noun} {entry.label!r} is referenced and cannot be deleted",
|
||||
category=ErrorCategory.CONFLICT,
|
||||
suggestion=(
|
||||
f"Deactivate the {self.short_noun} instead; "
|
||||
f"{self.referenced_retainer} will retain it."
|
||||
),
|
||||
)
|
||||
await _session.delete(entry)
|
||||
await self._finalize(session=_session, caller_session=session)
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Canonical Source media format policy.
|
||||
|
||||
Shared by every layer that needs to know which Source formats exist and what
|
||||
MIME type each maps to. Kept free of service classes and of any service-specific
|
||||
error type so no service module has to import a sibling service to use it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
SOURCE_MIME_TYPES = {
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".png": "image/png",
|
||||
".tif": "image/tiff",
|
||||
".tiff": "image/tiff",
|
||||
".pdf": "application/pdf",
|
||||
}
|
||||
SOURCE_EXTENSIONS = frozenset(SOURCE_MIME_TYPES)
|
||||
|
||||
|
||||
def lookup_source_mime_type(filename: str | Path) -> str | None:
|
||||
"""Return the canonical MIME type for a filename, or ``None`` if unsupported."""
|
||||
return SOURCE_MIME_TYPES.get(Path(filename).suffix.lower())
|
||||
|
||||
|
||||
def supported_source_formats() -> str:
|
||||
"""Return the supported Source extensions as a sorted display string."""
|
||||
return ", ".join(sorted(SOURCE_EXTENSIONS))
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import hashlib
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Sequence
|
||||
@@ -13,6 +13,7 @@ from dataclasses import dataclass
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -23,9 +24,11 @@ from pydantic import JsonValue
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import ValidationError
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy import inspect as sqlalchemy_inspect
|
||||
from sqlalchemy import literal
|
||||
from sqlalchemy import tuple_
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
from sqlalchemy.orm import defer
|
||||
from sqlalchemy.orm import selectinload
|
||||
from sqlmodel import col
|
||||
from sqlmodel import select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
@@ -51,25 +54,21 @@ from transcription.providers import TransportEvidence
|
||||
from transcription.providers import get_transcription_provider
|
||||
from transcription.providers.evidence import canonical_json_bytes
|
||||
|
||||
from ..db.loading import defer
|
||||
from ..db.loading import orm_attribute
|
||||
from ..db.loading import selectinload
|
||||
from .base import ServiceBase
|
||||
from .normalization import ORIENTATION_PRODUCER
|
||||
from .normalization import ORIENTATION_PRODUCER_VERSION
|
||||
from .normalization import ORIENTATION_SCHEMA
|
||||
from .normalization import ORIENTATION_SCHEMA_VERSION
|
||||
from .normalization import normalize_orientation
|
||||
from .normalization import normalize_orientation_async
|
||||
from .source_media import lookup_source_mime_type
|
||||
from .source_media import supported_source_formats
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_PROMPT_FILE = "transcribe_document.md"
|
||||
SOURCE_MIME_TYPES = {
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".png": "image/png",
|
||||
".tif": "image/tiff",
|
||||
".tiff": "image/tiff",
|
||||
".pdf": "application/pdf",
|
||||
}
|
||||
SOURCE_EXTENSIONS = frozenset(SOURCE_MIME_TYPES)
|
||||
JSON_OBJECT_ADAPTER = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
@@ -128,6 +127,14 @@ class ProviderInput:
|
||||
transformation: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LatestExecutionAttempt:
|
||||
"""One execution attempt plus the loader facts a caller needs to render it."""
|
||||
|
||||
attempt: ExecutionAttempt
|
||||
transport_body_deferred: bool
|
||||
|
||||
|
||||
class SourceService(ServiceBase):
|
||||
"""Manage source records, media payloads, revisions, and page execution output."""
|
||||
|
||||
@@ -154,6 +161,24 @@ class SourceService(ServiceBase):
|
||||
await close()
|
||||
self._provider = None
|
||||
|
||||
async def _read_source(
|
||||
self,
|
||||
*,
|
||||
session: AsyncSession,
|
||||
source_id: UUID,
|
||||
options: Sequence[Any] = (),
|
||||
suggestion: str = "Verify the source id and retry.",
|
||||
) -> Source:
|
||||
return await self._get_or_raise(
|
||||
Source,
|
||||
source_id,
|
||||
session=session,
|
||||
error=TranscriptionNotFoundError,
|
||||
noun="Source",
|
||||
suggestion=suggestion,
|
||||
options=options,
|
||||
)
|
||||
|
||||
async def create_source(self, source: Source, *, session: AsyncSession | None = None) -> Source:
|
||||
"""Create a new source page record in the database."""
|
||||
async with self._session_scope(session) as _session:
|
||||
@@ -164,14 +189,7 @@ class SourceService(ServiceBase):
|
||||
async def read_source(self, source_id: UUID, *, session: AsyncSession | None = None) -> Source:
|
||||
"""Read an existing source page record."""
|
||||
async with self._session_scope(session) as _session:
|
||||
source = await _session.get(Source, source_id)
|
||||
if source is None:
|
||||
raise TranscriptionNotFoundError(
|
||||
f"Source with id {source_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Verify the source id and retry.",
|
||||
)
|
||||
return source
|
||||
return await self._read_source(session=_session, source_id=source_id)
|
||||
|
||||
async def read_source_detail(self, source_id: UUID, *, session: AsyncSession | None = None) -> Source:
|
||||
"""Read a source page record with job-source context for UI detail rendering."""
|
||||
@@ -179,8 +197,8 @@ class SourceService(ServiceBase):
|
||||
query = (
|
||||
select(Source)
|
||||
.options(
|
||||
selectinload(Source.document), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Source.job_sources).selectinload(JobSource.job), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Source.document),
|
||||
selectinload(Source.job_sources).selectinload(orm_attribute(JobSource.job)),
|
||||
)
|
||||
.where(Source.id == source_id)
|
||||
.execution_options(populate_existing=True)
|
||||
@@ -200,20 +218,29 @@ class SourceService(ServiceBase):
|
||||
*,
|
||||
job_source_id: UUID,
|
||||
session: AsyncSession | None = None,
|
||||
) -> ExecutionAttempt | None:
|
||||
"""Read only the latest immutable attempt for one compatibility projection."""
|
||||
) -> LatestExecutionAttempt | None:
|
||||
"""Read only the latest immutable attempt for one compatibility projection.
|
||||
|
||||
The transport body is deferred because it can be arbitrarily large; the
|
||||
returned read model reports that as a plain flag so callers never have to
|
||||
inspect ORM loader state.
|
||||
"""
|
||||
async with self._session_scope(session) as _session:
|
||||
query = (
|
||||
select(ExecutionAttempt)
|
||||
.options(defer(ExecutionAttempt.transport_body)) # pyright: ignore[reportArgumentType]
|
||||
.options(defer(ExecutionAttempt.transport_body))
|
||||
.where(ExecutionAttempt.job_source_id == job_source_id)
|
||||
.order_by(
|
||||
ExecutionAttempt.attempt_number.desc(), # pyright: ignore[reportAttributeAccessIssue]
|
||||
ExecutionAttempt.id.desc(), # pyright: ignore[reportAttributeAccessIssue]
|
||||
col(ExecutionAttempt.attempt_number).desc(),
|
||||
col(ExecutionAttempt.id).desc(),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
return (await _session.exec(query)).first()
|
||||
attempt = (await _session.exec(query)).first()
|
||||
if attempt is None:
|
||||
return None
|
||||
deferred = "transport_body" in sqlalchemy_inspect(attempt).unloaded
|
||||
return LatestExecutionAttempt(attempt=attempt, transport_body_deferred=deferred)
|
||||
|
||||
async def read_source_navigation(
|
||||
self,
|
||||
@@ -223,24 +250,28 @@ class SourceService(ServiceBase):
|
||||
) -> SourceNavigation:
|
||||
"""Return adjacent Sources ordered within the current Document."""
|
||||
async with self._session_scope(session) as _session:
|
||||
source = await _session.get(Source, source_id)
|
||||
if source is None:
|
||||
raise TranscriptionNotFoundError(
|
||||
f"Source with id {source_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Verify the source id and retry.",
|
||||
source = await self._read_source(session=_session, source_id=source_id)
|
||||
position = (col(Source.page_number), col(Source.id))
|
||||
current = (literal(source.page_number), literal(source_id))
|
||||
|
||||
previous_query = (
|
||||
select(col(Source.id))
|
||||
.where(col(Source.document_id) == source.document_id)
|
||||
.where(tuple_(*position) < tuple_(*current))
|
||||
.order_by(col(Source.page_number).desc(), col(Source.id).desc())
|
||||
.limit(1)
|
||||
)
|
||||
query = (
|
||||
select(Source.id)
|
||||
.where(Source.document_id == source.document_id)
|
||||
.order_by(Source.page_number, Source.id) # pyright: ignore[reportArgumentType]
|
||||
next_query = (
|
||||
select(col(Source.id))
|
||||
.where(col(Source.document_id) == source.document_id)
|
||||
.where(tuple_(*position) > tuple_(*current))
|
||||
.order_by(col(Source.page_number), col(Source.id))
|
||||
.limit(1)
|
||||
)
|
||||
source_ids = list((await _session.exec(query)).all())
|
||||
|
||||
current_index = source_ids.index(source_id)
|
||||
return SourceNavigation(
|
||||
previous_id=source_ids[current_index - 1] if current_index > 0 else None,
|
||||
next_id=source_ids[current_index + 1] if current_index + 1 < len(source_ids) else None,
|
||||
previous_id=(await _session.exec(previous_query)).first(),
|
||||
next_id=(await _session.exec(next_query)).first(),
|
||||
)
|
||||
|
||||
async def update_source(self, source: Source, *, session: AsyncSession | None = None) -> Source:
|
||||
@@ -257,20 +288,14 @@ class SourceService(ServiceBase):
|
||||
async def delete_unlinked_source(self, *, source_id: UUID, session: AsyncSession | None = None) -> None:
|
||||
"""Delete a source only when no JobSource links exist."""
|
||||
async with self._session_scope(session) as _session:
|
||||
source = await _session.get(
|
||||
Source,
|
||||
source_id,
|
||||
source = await self._read_source(
|
||||
session=_session,
|
||||
source_id=source_id,
|
||||
options=(
|
||||
selectinload(Source.job_sources), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Source.processing_artifacts), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Source.job_sources),
|
||||
selectinload(Source.processing_artifacts),
|
||||
),
|
||||
)
|
||||
if source is None:
|
||||
raise TranscriptionNotFoundError(
|
||||
f"Source with id {source_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Verify the source id and retry.",
|
||||
)
|
||||
|
||||
if source.job_sources or source.processing_artifacts:
|
||||
raise SourceDeleteBlockedError(
|
||||
@@ -331,18 +356,12 @@ class SourceService(ServiceBase):
|
||||
)
|
||||
if document_id is not None:
|
||||
query = query.where(Source.document_id == document_id)
|
||||
|
||||
result = await _session.exec(query)
|
||||
sources = list(result.all())
|
||||
|
||||
if job_id is not None:
|
||||
sources = [
|
||||
source
|
||||
for source in sources
|
||||
if any(job_source.job_id == job_id for job_source in source.job_sources)
|
||||
]
|
||||
query = query.join(JobSource, col(JobSource.source_id) == col(Source.id)).where(
|
||||
col(JobSource.job_id) == job_id
|
||||
)
|
||||
|
||||
return sources
|
||||
return list((await _session.exec(query)).all())
|
||||
|
||||
async def create_job_source(
|
||||
self,
|
||||
@@ -362,7 +381,7 @@ class SourceService(ServiceBase):
|
||||
job_source = await _session.get(
|
||||
JobSource,
|
||||
job_source_id,
|
||||
options=(selectinload(JobSource.source),), # pyright: ignore[reportArgumentType]
|
||||
options=(selectinload(JobSource.source),),
|
||||
)
|
||||
if job_source is None:
|
||||
raise TranscriptionNotFoundError(
|
||||
@@ -423,20 +442,14 @@ class SourceService(ServiceBase):
|
||||
- Blocked when additional JobSource links exist (history/shared dependencies).
|
||||
"""
|
||||
async with self._session_scope(session) as _session:
|
||||
source = await _session.get(
|
||||
Source,
|
||||
source_id,
|
||||
source = await self._read_source(
|
||||
session=_session,
|
||||
source_id=source_id,
|
||||
options=(
|
||||
selectinload(Source.job_sources), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Source.processing_artifacts), # pyright: ignore[reportArgumentType]
|
||||
selectinload(Source.job_sources),
|
||||
selectinload(Source.processing_artifacts),
|
||||
),
|
||||
)
|
||||
if source is None:
|
||||
raise TranscriptionNotFoundError(
|
||||
f"Source with id {source_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Verify the source id and retry.",
|
||||
)
|
||||
|
||||
linked_job_sources = list(source.job_sources)
|
||||
attempt_count = (
|
||||
@@ -500,8 +513,8 @@ class SourceService(ServiceBase):
|
||||
"""List job-source records, optionally filtered by job."""
|
||||
async with self._session_scope(session) as _session:
|
||||
query = select(JobSource).options(
|
||||
selectinload(JobSource.job), # pyright: ignore[reportArgumentType]
|
||||
selectinload(JobSource.source), # pyright: ignore[reportArgumentType]
|
||||
selectinload(JobSource.job),
|
||||
selectinload(JobSource.source),
|
||||
)
|
||||
if job_id is not None:
|
||||
query = query.where(JobSource.job_id == job_id)
|
||||
@@ -531,21 +544,16 @@ class SourceService(ServiceBase):
|
||||
) -> JobSource:
|
||||
"""Persist transcription fields for one source within a specific job."""
|
||||
async with self._session_scope(session) as _session:
|
||||
job = await _session.get(Job, job_id)
|
||||
if job is None:
|
||||
raise TranscriptionNotFoundError(
|
||||
f"Job with id {job_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
job = await self._get_or_raise(
|
||||
Job,
|
||||
job_id,
|
||||
session=_session,
|
||||
error=TranscriptionNotFoundError,
|
||||
noun="Job",
|
||||
suggestion="Verify the job id and retry.",
|
||||
)
|
||||
|
||||
source = await _session.get(Source, source_id)
|
||||
if source is None:
|
||||
raise TranscriptionNotFoundError(
|
||||
f"Source with id {source_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Verify the source id and retry.",
|
||||
)
|
||||
source = await self._read_source(session=_session, source_id=source_id)
|
||||
|
||||
if source.document_id != job.document_id:
|
||||
raise TranscriptionError(
|
||||
@@ -556,7 +564,6 @@ class SourceService(ServiceBase):
|
||||
|
||||
job.provider = provider or job.provider or self.settings.provider.value
|
||||
job.model = model or job.model or _resolve_transcript_model(provider=self.provider, settings=self.settings)
|
||||
job.date_updated = datetime.now(UTC)
|
||||
metadata_payload = _validate_transcription_metadata(ai_metadata)
|
||||
raw_response_payload = _validate_json_object(raw_api_response, field_name="raw_api_response")
|
||||
|
||||
@@ -678,11 +685,9 @@ class SourceService(ServiceBase):
|
||||
) -> Source:
|
||||
"""Atomically select one successful machine attempt as the Source projection."""
|
||||
async with self._session_scope(session) as _session:
|
||||
source = await _session.get(Source, source_id)
|
||||
if source is None:
|
||||
raise TranscriptionNotFoundError(
|
||||
f"Source with id {source_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
source = await self._read_source(
|
||||
session=_session,
|
||||
source_id=source_id,
|
||||
suggestion="Refresh Source Detail and retry.",
|
||||
)
|
||||
attempt = await _session.get(ExecutionAttempt, execution_attempt_id)
|
||||
@@ -717,10 +722,10 @@ class SourceService(ServiceBase):
|
||||
if job_id is not None:
|
||||
query = query.where(ExecutionAttempt.job_id == job_id)
|
||||
query = query.order_by(
|
||||
ExecutionAttempt.job_id,
|
||||
ExecutionAttempt.source_id,
|
||||
ExecutionAttempt.attempt_number,
|
||||
ExecutionAttempt.id,
|
||||
col(ExecutionAttempt.job_id),
|
||||
col(ExecutionAttempt.source_id),
|
||||
col(ExecutionAttempt.attempt_number),
|
||||
col(ExecutionAttempt.id),
|
||||
)
|
||||
return (await _session.exec(query)).all()
|
||||
|
||||
@@ -775,7 +780,7 @@ class SourceService(ServiceBase):
|
||||
if len(payload_bytes) > self.settings.artifact_inline_threshold_bytes:
|
||||
relative_path = Path(str(source_id)) / f"{artifact_id}.json"
|
||||
external_path = self.settings.artifact_dir / relative_path
|
||||
self._write_external_artifact(path=external_path, content=payload_bytes)
|
||||
await asyncio.to_thread(self._write_external_artifact, path=external_path, content=payload_bytes)
|
||||
inline_payload = None
|
||||
external_reference = relative_path.as_posix()
|
||||
artifact = ProcessingArtifact(
|
||||
@@ -821,7 +826,7 @@ class SourceService(ServiceBase):
|
||||
safe_suffix = suffix if suffix.startswith(".") and suffix[1:].isalnum() else ".bin"
|
||||
relative_path = Path(str(source_id)) / f"{artifact_id}{safe_suffix.lower()}"
|
||||
external_path = self.settings.artifact_dir / relative_path
|
||||
self._write_external_artifact(path=external_path, content=content)
|
||||
payload_sha256 = await asyncio.to_thread(self._write_and_digest_artifact, path=external_path, content=content)
|
||||
artifact = ProcessingArtifact(
|
||||
id=artifact_id,
|
||||
source_id=source_id,
|
||||
@@ -832,7 +837,7 @@ class SourceService(ServiceBase):
|
||||
producer=producer,
|
||||
producer_version=producer_version,
|
||||
external_reference=relative_path.as_posix(),
|
||||
payload_sha256=hashlib.sha256(content).hexdigest(),
|
||||
payload_sha256=payload_sha256,
|
||||
byte_size=len(content),
|
||||
coordinate_metadata=coordinate_metadata,
|
||||
)
|
||||
@@ -850,7 +855,7 @@ class SourceService(ServiceBase):
|
||||
) -> ProviderInput:
|
||||
"""Resolve original or physically orientation-normalized provider input."""
|
||||
media_type = source_mime_type(source.file_path)
|
||||
normalized = normalize_orientation(source.file_path, media_type=media_type)
|
||||
normalized = await normalize_orientation_async(source.file_path, media_type=media_type)
|
||||
if normalized is None:
|
||||
return ProviderInput(
|
||||
path=Path(source.file_path),
|
||||
@@ -905,6 +910,15 @@ class SourceService(ServiceBase):
|
||||
transformation=f"{ORIENTATION_SCHEMA}@{ORIENTATION_SCHEMA_VERSION}",
|
||||
)
|
||||
|
||||
def _write_and_digest_artifact(self, *, path: Path, content: bytes) -> str:
|
||||
"""Persist artifact bytes and return their digest in one off-loop hop.
|
||||
|
||||
Binary derivatives are page-sized, so hashing them belongs in the same
|
||||
worker thread as the write rather than on the event loop ([MED-01]).
|
||||
"""
|
||||
self._write_external_artifact(path=path, content=content)
|
||||
return hashlib.sha256(content).hexdigest()
|
||||
|
||||
def _write_external_artifact(self, *, path: Path, content: bytes) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary_path = path.with_suffix(f"{path.suffix}.tmp")
|
||||
@@ -946,6 +960,11 @@ class SourceService(ServiceBase):
|
||||
suggestion="Restore the expected artifact bytes before retrying.",
|
||||
)
|
||||
|
||||
def _verify_artifacts_integrity(self, artifacts: Sequence[ProcessingArtifact]) -> None:
|
||||
"""Verify a batch of artifacts; hashing and file reads run off the event loop."""
|
||||
for artifact in artifacts:
|
||||
self._verify_artifact_integrity(artifact)
|
||||
|
||||
def _verify_artifact_integrity(self, artifact: ProcessingArtifact) -> None:
|
||||
if artifact.inline_payload is None:
|
||||
self._verify_external_artifact(artifact)
|
||||
@@ -962,6 +981,7 @@ class SourceService(ServiceBase):
|
||||
self,
|
||||
*,
|
||||
source_id: UUID,
|
||||
limit: int = 100,
|
||||
session: AsyncSession | None = None,
|
||||
) -> Sequence[ProcessingArtifact]:
|
||||
"""List generic artifacts associated with a Source."""
|
||||
@@ -969,7 +989,8 @@ class SourceService(ServiceBase):
|
||||
query = (
|
||||
select(ProcessingArtifact)
|
||||
.where(ProcessingArtifact.source_id == source_id)
|
||||
.order_by(ProcessingArtifact.created_at, ProcessingArtifact.id)
|
||||
.order_by(col(ProcessingArtifact.created_at), col(ProcessingArtifact.id))
|
||||
.limit(limit)
|
||||
)
|
||||
return (await _session.exec(query)).all()
|
||||
|
||||
@@ -984,9 +1005,9 @@ class SourceService(ServiceBase):
|
||||
async with self._session_scope(session) as _session:
|
||||
query = (
|
||||
select(ProcessingArtifact)
|
||||
.options(defer(ProcessingArtifact.inline_payload)) # pyright: ignore[reportArgumentType]
|
||||
.options(defer(ProcessingArtifact.inline_payload))
|
||||
.where(ProcessingArtifact.source_id == source_id)
|
||||
.order_by(ProcessingArtifact.created_at, ProcessingArtifact.id)
|
||||
.order_by(col(ProcessingArtifact.created_at), col(ProcessingArtifact.id))
|
||||
.limit(limit)
|
||||
)
|
||||
return (await _session.exec(query)).all()
|
||||
@@ -999,18 +1020,11 @@ class SourceService(ServiceBase):
|
||||
) -> dict[str, JsonValue]:
|
||||
"""Build a versioned, source-reference-only evidence export."""
|
||||
async with self._session_scope(session) as _session:
|
||||
source = await _session.get(Source, source_id)
|
||||
if source is None:
|
||||
raise TranscriptionNotFoundError(
|
||||
f"Source with id {source_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Verify the source id and retry.",
|
||||
)
|
||||
source = await self._read_source(session=_session, source_id=source_id)
|
||||
attempts = list(await self.list_execution_attempts(source_id=source_id, session=_session))
|
||||
artifacts = list(await self.list_processing_artifacts(source_id=source_id, session=_session))
|
||||
|
||||
for artifact in artifacts:
|
||||
self._verify_artifact_integrity(artifact)
|
||||
await asyncio.to_thread(self._verify_artifacts_integrity, artifacts)
|
||||
|
||||
artifact_payloads = [
|
||||
{
|
||||
@@ -1099,13 +1113,7 @@ class SourceService(ServiceBase):
|
||||
) -> Source:
|
||||
"""Persist a human revision on a source page."""
|
||||
async with self._session_scope(session) as _session:
|
||||
source = await _session.get(Source, source_id)
|
||||
if source is None:
|
||||
raise TranscriptionNotFoundError(
|
||||
f"Source with id {source_id} not found",
|
||||
category=ErrorCategory.NOT_FOUND,
|
||||
suggestion="Verify the source id and retry.",
|
||||
)
|
||||
source = await self._read_source(session=_session, source_id=source_id)
|
||||
|
||||
source.revised_text = text
|
||||
source.date_revised = datetime.now(UTC)
|
||||
@@ -1132,18 +1140,18 @@ class SourceService(ServiceBase):
|
||||
async with self._session_scope(session) as _session:
|
||||
query = (
|
||||
select(Source)
|
||||
.join(JobSource, JobSource.source_id == Source.id)
|
||||
.where(JobSource.job_id == job_id)
|
||||
.where(Source.revised_text.is_not(None))
|
||||
.order_by(Source.date_revised) # pyright: ignore[reportArgumentType]
|
||||
.join(JobSource, col(JobSource.source_id) == col(Source.id))
|
||||
.where(col(JobSource.job_id) == job_id)
|
||||
.where(col(Source.revised_text).is_not(None))
|
||||
.order_by(col(Source.date_revised))
|
||||
)
|
||||
result = await _session.exec(query)
|
||||
return result.all()
|
||||
|
||||
|
||||
def _resolve_transcript_model(*, provider: TranscriptionProvider, settings: Settings) -> str:
|
||||
provider_model = getattr(provider, "model", None)
|
||||
if isinstance(provider_model, str) and provider_model.strip():
|
||||
provider_model = provider.model
|
||||
if provider_model and provider_model.strip():
|
||||
return provider_model
|
||||
|
||||
if settings.provider_model and settings.provider_model.strip():
|
||||
@@ -1207,7 +1215,9 @@ async def transcribe_document_image(
|
||||
"""Transcribe a local image using the configured prompt and provider."""
|
||||
runtime_settings = settings or get_settings()
|
||||
if prompt_text is None:
|
||||
prompt_execution = build_prompt_execution(prompt_name=prompt_name, settings=runtime_settings)
|
||||
prompt_execution = await asyncio.to_thread(
|
||||
build_prompt_execution, prompt_name=prompt_name, settings=runtime_settings
|
||||
)
|
||||
else:
|
||||
effective_prompt_name = (prompt_name or runtime_settings.default_prompt_name or DEFAULT_PROMPT_FILE).strip()
|
||||
prompt_execution = PromptExecution(
|
||||
@@ -1218,7 +1228,7 @@ async def transcribe_document_image(
|
||||
temperature=temperature if temperature is not None else runtime_settings.transcription_temperature,
|
||||
top_p=top_p if top_p is not None else runtime_settings.transcription_top_p,
|
||||
)
|
||||
image_bytes, mime_type = load_source_payload(image_path)
|
||||
image_bytes, mime_type = await asyncio.to_thread(load_source_payload, image_path)
|
||||
|
||||
owns_adapter = provider is None
|
||||
adapter = provider or get_transcription_provider(settings=runtime_settings)
|
||||
@@ -1226,22 +1236,18 @@ async def transcribe_document_image(
|
||||
|
||||
try:
|
||||
with handle_transcription_errors():
|
||||
transcribe_kwargs = {
|
||||
"prompt_text": prompt_execution.user_prompt,
|
||||
"image_bytes": image_bytes,
|
||||
"mime_type": mime_type,
|
||||
"temperature": prompt_execution.temperature,
|
||||
"top_p": prompt_execution.top_p,
|
||||
"source_reference": source_reference,
|
||||
}
|
||||
if "requested_model" in inspect.signature(adapter.transcribe).parameters:
|
||||
transcribe_kwargs["requested_model"] = requested_model
|
||||
result = await adapter.transcribe(**transcribe_kwargs)
|
||||
result = await adapter.transcribe(
|
||||
prompt_text=prompt_execution.user_prompt,
|
||||
image_bytes=image_bytes,
|
||||
mime_type=mime_type,
|
||||
temperature=prompt_execution.temperature,
|
||||
top_p=prompt_execution.top_p,
|
||||
source_reference=source_reference,
|
||||
requested_model=requested_model,
|
||||
)
|
||||
finally:
|
||||
if owns_adapter:
|
||||
close = getattr(adapter, "aclose", None)
|
||||
if close is not None:
|
||||
await close()
|
||||
await adapter.aclose()
|
||||
logger.info("Transcription completed for image=%s provider=%s", image_path, result.provider)
|
||||
return TranscriptionResult(
|
||||
text=result.text,
|
||||
@@ -1308,15 +1314,13 @@ def load_prompt_text(*, prompt_name: str = DEFAULT_PROMPT_FILE, settings: Settin
|
||||
|
||||
def source_mime_type(filename: str | Path) -> str:
|
||||
"""Return the canonical MIME type for a supported Source filename."""
|
||||
path = Path(filename)
|
||||
suffix = path.suffix.lower()
|
||||
mime_type = SOURCE_MIME_TYPES.get(suffix)
|
||||
mime_type = lookup_source_mime_type(filename)
|
||||
if mime_type is None:
|
||||
supported = ", ".join(sorted(SOURCE_EXTENSIONS))
|
||||
suffix = Path(filename).suffix.lower()
|
||||
raise TranscriptionError(
|
||||
f"Unsupported Source format: {suffix or '<none>'}",
|
||||
category=ErrorCategory.USER_INPUT,
|
||||
suggestion=f"Use one of the supported Source formats: {supported}.",
|
||||
suggestion=f"Use one of the supported Source formats: {supported_source_formats()}.",
|
||||
)
|
||||
return mime_type
|
||||
|
||||
@@ -1341,7 +1345,10 @@ def validate_source_content(*, filename: str | Path, content: bytes) -> str:
|
||||
|
||||
|
||||
def load_source_payload(source_path: str | Path) -> tuple[bytes, str]:
|
||||
"""Read Source bytes and resolve MIME type from the canonical format policy."""
|
||||
"""Read Source bytes and resolve MIME type from the canonical format policy.
|
||||
|
||||
Blocking. Async callers must dispatch this through ``asyncio.to_thread``.
|
||||
"""
|
||||
path = Path(source_path)
|
||||
|
||||
if not path.exists() or not path.is_file():
|
||||
|
||||
@@ -21,9 +21,13 @@ from ..db.models import Job
|
||||
from ..db.models import JobSource
|
||||
from ..db.models import JobSourceStatus
|
||||
from ..db.models import Source
|
||||
from ..db.session import SessionFactory
|
||||
from ..db.session import session_scope
|
||||
from .media_storage import build_stored_filename
|
||||
from .media_storage import write_media_bytes
|
||||
from .sources import TranscriptionError
|
||||
from .sources import build_prompt_execution
|
||||
from .sources import validate_source_content
|
||||
from .transcription import build_prompt_execution
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -32,9 +36,6 @@ class SourceStorageError(AppError):
|
||||
"""Raised when Source content cannot be validated or persisted safely."""
|
||||
|
||||
|
||||
UploadError = SourceStorageError
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class JobCreateResult:
|
||||
"""Summary of explicit Job create records."""
|
||||
@@ -69,15 +70,20 @@ async def create_document_job(
|
||||
*,
|
||||
filename: str,
|
||||
file_bytes: bytes,
|
||||
session: AsyncSession,
|
||||
session: AsyncSession | None = None,
|
||||
session_factory: SessionFactory | None = None,
|
||||
settings: Settings | None = None,
|
||||
) -> DocumentJobResult:
|
||||
"""Create a Document, its first Source, and a queued Job."""
|
||||
"""Create a Document, its first Source, and a queued Job.
|
||||
|
||||
Owns its own session when the caller does not supply one, so UI callers
|
||||
never have to import a session scope.
|
||||
"""
|
||||
runtime_settings = settings or get_settings()
|
||||
prompt_execution = build_prompt_execution(settings=runtime_settings)
|
||||
document_id = uuid4()
|
||||
source_id = uuid4()
|
||||
stored_path = store_source_file(
|
||||
stored_path = await store_source_file(
|
||||
filename=filename,
|
||||
file_bytes=file_bytes,
|
||||
settings=runtime_settings,
|
||||
@@ -86,8 +92,13 @@ async def create_document_job(
|
||||
)
|
||||
file_hash, file_size_bytes = _compute_file_metadata(file_bytes)
|
||||
try:
|
||||
document, job = await _create_document_job_records(
|
||||
async with session_scope(
|
||||
session_factory=session_factory,
|
||||
session=session,
|
||||
settings=runtime_settings,
|
||||
) as _session:
|
||||
document, job = await _create_document_job_records(
|
||||
session=_session,
|
||||
document_id=document_id,
|
||||
source_id=source_id,
|
||||
original_filename=filename,
|
||||
@@ -118,12 +129,17 @@ async def create_job_for_document(
|
||||
*,
|
||||
document_id: UUID,
|
||||
source_files: Sequence[tuple[str, bytes]],
|
||||
session: AsyncSession,
|
||||
session: AsyncSession | None = None,
|
||||
session_factory: SessionFactory | None = None,
|
||||
provider: str | None = None,
|
||||
model: str | None = None,
|
||||
settings: Settings | None = None,
|
||||
) -> JobCreateResult:
|
||||
"""Create a queued Job for an existing Document with one or more Sources."""
|
||||
"""Create a queued Job for an existing Document with one or more Sources.
|
||||
|
||||
Owns its own session when the caller does not supply one, so UI callers
|
||||
never have to import a session scope.
|
||||
"""
|
||||
if not source_files:
|
||||
raise SourceStorageError(
|
||||
"At least one Source file is required to create a Job",
|
||||
@@ -137,25 +153,31 @@ async def create_job_for_document(
|
||||
stored_sources: list[PendingStoredSource] = []
|
||||
for filename, file_bytes in sorted_source_files:
|
||||
source_id = uuid4()
|
||||
stored_sources.append(
|
||||
PendingStoredSource(
|
||||
source_id=source_id,
|
||||
original_filename=filename,
|
||||
stored_path=store_source_file(
|
||||
stored_path = await store_source_file(
|
||||
filename=filename,
|
||||
file_bytes=file_bytes,
|
||||
settings=runtime_settings,
|
||||
relative_directory=Path("documents") / str(document_id),
|
||||
filename_stem=str(source_id),
|
||||
),
|
||||
)
|
||||
stored_sources.append(
|
||||
PendingStoredSource(
|
||||
source_id=source_id,
|
||||
original_filename=filename,
|
||||
stored_path=stored_path,
|
||||
file_hash=_compute_file_hash(file_bytes),
|
||||
file_size_bytes=len(file_bytes),
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
job, source_ids = await _create_job_for_document_records(
|
||||
async with session_scope(
|
||||
session_factory=session_factory,
|
||||
session=session,
|
||||
settings=runtime_settings,
|
||||
) as _session:
|
||||
job, source_ids = await _create_job_for_document_records(
|
||||
session=_session,
|
||||
document_id=document_id,
|
||||
stored_sources=stored_sources,
|
||||
provider=provider,
|
||||
@@ -316,7 +338,7 @@ def _compute_file_metadata(file_bytes: bytes) -> tuple[str, int]:
|
||||
return _compute_file_hash(file_bytes), len(file_bytes)
|
||||
|
||||
|
||||
def store_source_file(
|
||||
async def store_source_file(
|
||||
*,
|
||||
filename: str,
|
||||
file_bytes: bytes,
|
||||
@@ -335,49 +357,14 @@ def store_source_file(
|
||||
suggestion=exc.suggestion,
|
||||
retriable=exc.retriable,
|
||||
) from exc
|
||||
return _store_file_bytes(
|
||||
filename=filename,
|
||||
|
||||
upload_dir = runtime_settings.upload_dir
|
||||
return await write_media_bytes(
|
||||
target_dir=upload_dir if relative_directory is None else upload_dir / relative_directory,
|
||||
stored_name=build_stored_filename(filename=filename, filename_stem=filename_stem),
|
||||
file_bytes=file_bytes,
|
||||
settings=runtime_settings,
|
||||
relative_directory=relative_directory,
|
||||
filename_stem=filename_stem,
|
||||
error=SourceStorageError,
|
||||
failure_message="Failed to persist Source file",
|
||||
failure_suggestion="Check upload directory permissions and available disk space, then retry.",
|
||||
log_label="Source file",
|
||||
)
|
||||
|
||||
|
||||
def _store_file_bytes(
|
||||
*,
|
||||
filename: str,
|
||||
file_bytes: bytes,
|
||||
settings: Settings,
|
||||
relative_directory: Path | None = None,
|
||||
filename_stem: str | None = None,
|
||||
) -> Path:
|
||||
upload_dir = settings.upload_dir
|
||||
target_dir = upload_dir if relative_directory is None else upload_dir / relative_directory
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
stored_name = _build_stored_filename(filename=filename, filename_stem=filename_stem)
|
||||
stored_path = target_dir / stored_name
|
||||
|
||||
try:
|
||||
stored_path.write_bytes(file_bytes)
|
||||
except OSError as exc:
|
||||
raise SourceStorageError(
|
||||
"Failed to persist Source file",
|
||||
category=ErrorCategory.INFRA_PERSISTENT,
|
||||
suggestion="Check upload directory permissions and available disk space, then retry.",
|
||||
) from exc
|
||||
|
||||
logger.info("Stored Source file: %s", stored_path)
|
||||
return stored_path
|
||||
|
||||
|
||||
def _build_stored_filename(*, filename: str, filename_stem: str | None = None) -> str:
|
||||
safe_name = Path(filename).name
|
||||
suffix = Path(safe_name).suffix.lower()
|
||||
stem = filename_stem or str(uuid4())
|
||||
return f"{stem}{suffix}"
|
||||
|
||||
|
||||
create_upload_job = create_document_job
|
||||
store_file = store_source_file
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
"""Compatibility exports for the Source-owned transcription implementation."""
|
||||
|
||||
from .sources import DEFAULT_PROMPT_FILE
|
||||
from .sources import SOURCE_EXTENSIONS
|
||||
from .sources import SOURCE_MIME_TYPES
|
||||
from .sources import PromptExecution
|
||||
from .sources import PromptLoadError
|
||||
from .sources import SourceDeleteBlockedError
|
||||
from .sources import SourceService
|
||||
from .sources import TranscriptionError
|
||||
from .sources import TranscriptionNotFoundError
|
||||
from .sources import build_prompt_execution
|
||||
from .sources import handle_transcription_errors
|
||||
from .sources import load_prompt_text
|
||||
from .sources import load_source_payload
|
||||
from .sources import source_mime_type
|
||||
from .sources import transcribe_document_image
|
||||
from .sources import validate_source_content
|
||||
|
||||
TranscriptionService = SourceService
|
||||
SUPPORTED_EXTENSIONS = SOURCE_EXTENSIONS
|
||||
load_image_payload = load_source_payload
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_PROMPT_FILE",
|
||||
"SOURCE_EXTENSIONS",
|
||||
"SOURCE_MIME_TYPES",
|
||||
"SUPPORTED_EXTENSIONS",
|
||||
"PromptExecution",
|
||||
"PromptLoadError",
|
||||
"SourceDeleteBlockedError",
|
||||
"SourceService",
|
||||
"TranscriptionError",
|
||||
"TranscriptionNotFoundError",
|
||||
"TranscriptionService",
|
||||
"build_prompt_execution",
|
||||
"handle_transcription_errors",
|
||||
"load_image_payload",
|
||||
"load_prompt_text",
|
||||
"load_source_payload",
|
||||
"source_mime_type",
|
||||
"transcribe_document_image",
|
||||
"validate_source_content",
|
||||
]
|
||||
@@ -185,13 +185,15 @@ async def process_queued_job( # noqa: PLR0915
|
||||
logger.warning(f"Job {job.id} is not queued. Current status: {job.status}")
|
||||
return
|
||||
|
||||
# Transaction A: claim job for processing.
|
||||
# Transaction A: claim job for processing. Reached only when a caller hands us a
|
||||
# still-QUEUED job directly; the worker path already claimed it atomically in
|
||||
# JobService.claim_next_queued_job.
|
||||
if current_status == JobStatus.QUEUED:
|
||||
if session is None:
|
||||
job = await services.jobs.mark_job_status(job.id, JobStatus.PROCESSING)
|
||||
else:
|
||||
# If we're sharing the session, need to make sure setting the Job to PROCESSING is committed before we start
|
||||
# the transcription, otherwise other workers may see the job as still QUEUED and try to process it.
|
||||
# Commit the PROCESSING transition before transcription starts so the claim
|
||||
# is durable and visible to any other worker before the long provider call.
|
||||
job = await services.jobs.mark_job_status(job.id, JobStatus.PROCESSING, session=session)
|
||||
await session.commit()
|
||||
|
||||
@@ -294,12 +296,8 @@ async def process_queued_job( # noqa: PLR0915
|
||||
0,
|
||||
int((asyncio.get_running_loop().time() - monotonic_started_at) * 1000),
|
||||
),
|
||||
request_manifest=getattr(services.sources.provider, "current_request_manifest", None),
|
||||
transport_evidence=getattr(
|
||||
services.sources.provider,
|
||||
"current_transport_evidence",
|
||||
None,
|
||||
),
|
||||
request_manifest=services.sources.provider.current_request_manifest,
|
||||
transport_evidence=services.sources.provider.current_transport_evidence,
|
||||
failure_phase="local_timeout",
|
||||
model_input_artifact_id=(
|
||||
provider_input.derivative_id if provider_input is not None else None
|
||||
@@ -417,11 +415,16 @@ async def process_next_queued_job(
|
||||
session: AsyncSession | None = None,
|
||||
) -> bool:
|
||||
"""Process the next queued job if one exists."""
|
||||
job = await services.jobs.read_next_queued_job(session=session)
|
||||
job = await services.jobs.claim_next_queued_job(session=session)
|
||||
|
||||
if job is None:
|
||||
return False
|
||||
|
||||
# The claim must be durable before the provider call starts, otherwise another
|
||||
# worker could observe the job as still QUEUED and process it a second time.
|
||||
if session is not None:
|
||||
await session.commit()
|
||||
|
||||
await advance_job(job=job, services=services, settings=settings, session=session)
|
||||
return True
|
||||
|
||||
@@ -557,7 +560,7 @@ async def _write_page_outcome(
|
||||
warnings = analyze_transcription_quality(result.text)
|
||||
await services.sources.create_json_artifact(
|
||||
source_id=source.id,
|
||||
execution_attempt_id=attempt.id,
|
||||
execution_attempt_id=attempt.attempt.id,
|
||||
artifact_type="transcription_quality_warnings",
|
||||
schema_name=QUALITY_ANALYSIS_SCHEMA,
|
||||
schema_version=QUALITY_ANALYSIS_VERSION,
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
from transcription.ui.components.app_shell import NAV_ITEMS
|
||||
from transcription.ui.components.app_shell import render_app_shell
|
||||
from transcription.ui.components.app_shell import render_navigation_header
|
||||
from transcription.ui.components.document_panzoom import render_document_panzoom
|
||||
from transcription.ui.components.primitives import destructive_button
|
||||
from transcription.ui.components.primitives import render_empty_state
|
||||
from transcription.ui.components.primitives import section_header_row
|
||||
@@ -12,7 +11,6 @@ __all__ = [
|
||||
"NAV_ITEMS",
|
||||
"destructive_button",
|
||||
"render_app_shell",
|
||||
"render_document_panzoom",
|
||||
"render_empty_state",
|
||||
"render_navigation_header",
|
||||
"section_header_row",
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from nicegui import ui
|
||||
|
||||
from transcription.ui.theme import VIBESCRIBE_LOGO_SVG
|
||||
from transcription.ui.resources import read_svg
|
||||
|
||||
NAV_ITEMS: tuple[tuple[str, str, str], ...] = (
|
||||
("Documents", "/documents", "description"),
|
||||
@@ -55,7 +55,7 @@ def render_app_shell(*, current_path: str | None = None) -> None:
|
||||
|
||||
with ui.header().classes("app-shell"), ui.element("div").classes("app-shell__inner"):
|
||||
with ui.element("a").props('href="/ui/homepage"').classes("app-shell__brand no-wrap"):
|
||||
ui.html(VIBESCRIBE_LOGO_SVG).classes("app-shell__brand-mark")
|
||||
ui.html(read_svg("vibescribe_logo.svg")).classes("app-shell__brand-mark")
|
||||
ui.label("VibeScribe").classes("app-shell__brand-name")
|
||||
|
||||
with ui.element("nav").props('aria-label="Primary navigation"').classes("app-shell__nav"):
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Shared rendering for the destructive-confirmation pages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Sequence
|
||||
|
||||
from nicegui import ui
|
||||
|
||||
from transcription.ui.components.primitives import destructive_button
|
||||
|
||||
|
||||
def render_delete_blocked_notice(
|
||||
*,
|
||||
reason: str,
|
||||
guidance: str,
|
||||
detail: str | None = None,
|
||||
back_label: str,
|
||||
back_target: str,
|
||||
secondary_label: str = "Go to Jobs",
|
||||
secondary_target: str = "/jobs",
|
||||
secondary_icon: str = "work_history",
|
||||
) -> None:
|
||||
"""Render the blocked-dependency notice plus its two navigation actions."""
|
||||
ui.label(reason).classes("text-xs ui-text-danger font-bold mt-2")
|
||||
if detail is not None:
|
||||
ui.label(detail).classes("text-xs ui-text-muted")
|
||||
ui.label(guidance).classes("text-xs ui-text-muted italic")
|
||||
with ui.row().classes("w-full items-center gap-2 mt-4"):
|
||||
ui.button(back_label, on_click=lambda: ui.navigate.to(back_target), icon="arrow_back").classes(
|
||||
"ui-btn-primary text-xs"
|
||||
)
|
||||
ui.button(
|
||||
secondary_label,
|
||||
on_click=lambda: ui.navigate.to(secondary_target),
|
||||
icon=secondary_icon,
|
||||
).props("flat text-xs")
|
||||
|
||||
|
||||
def render_delete_actions(
|
||||
*,
|
||||
confirm_label: str,
|
||||
on_confirm: Callable[[], Awaitable[None]],
|
||||
cancel_target: str,
|
||||
) -> None:
|
||||
"""Render the permanent-delete button beside a cancel action."""
|
||||
with ui.row().classes("w-full items-center gap-2 mt-2"):
|
||||
destructive_button(confirm_label, on_click=on_confirm, icon="delete_forever", variant="solid")
|
||||
ui.button("Cancel", on_click=lambda: ui.navigate.to(cancel_target), icon="arrow_back").props("flat")
|
||||
|
||||
|
||||
def dependency_summary(categories: Sequence[tuple[str, bool]]) -> str:
|
||||
"""Describe which related record categories are blocking a delete."""
|
||||
present = [name for name, is_present in categories if is_present]
|
||||
return f"Dependencies present: {', '.join(present)}"
|
||||
@@ -1,169 +0,0 @@
|
||||
"""Panzoom-backed document preview component."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
from uuid import uuid4
|
||||
|
||||
from nicegui import ui
|
||||
|
||||
from transcription.config import get_settings
|
||||
from transcription.db.models import Source
|
||||
|
||||
PANGOZOOM_CDN_URL = "https://unpkg.com/@panzoom/[email protected]/dist/panzoom.min.js"
|
||||
UPLOADS_URL_PREFIX = "/uploads"
|
||||
|
||||
|
||||
def render_document_panzoom(*, source: Source) -> None:
|
||||
"""Render a source preview with pan and zoom interactions."""
|
||||
_register_panzoom_assets()
|
||||
|
||||
host_id = f"document-panzoom-{uuid4().hex}"
|
||||
document_url = _document_url(source)
|
||||
document_kind = _document_kind(source)
|
||||
|
||||
with ui.card().classes("w-full q-pa-md ui-card-surface"):
|
||||
with ui.row().classes("w-full items-center justify-between no-wrap"):
|
||||
ui.label("Document preview").classes("text-subtitle1 text-weight-medium")
|
||||
ui.label(source.filename).classes("text-caption ui-text-muted ellipsis document-panzoom-filename")
|
||||
|
||||
with ui.element("div").classes("w-full document-panzoom-host q-mt-md") as host:
|
||||
host.props(f"id={host_id}")
|
||||
with ui.element("div").classes("document-panzoom-surface"):
|
||||
if document_kind == "pdf":
|
||||
ui.html(
|
||||
f'<iframe class="document-panzoom-iframe" '
|
||||
f'src="{document_url}" title="{source.filename}" '
|
||||
"data-panzoom-target></iframe>"
|
||||
)
|
||||
else:
|
||||
ui.html(
|
||||
f'<img class="document-panzoom-media" '
|
||||
f'src="{document_url}" alt="{source.filename}" '
|
||||
"data-panzoom-target data-panzoom-media />"
|
||||
)
|
||||
|
||||
_attach_panzoom(host_id)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _register_panzoom_assets() -> None:
|
||||
ui.add_head_html(
|
||||
f'<script src="{PANGOZOOM_CDN_URL}"></script>',
|
||||
shared=True,
|
||||
)
|
||||
|
||||
|
||||
def _document_url(source: Source) -> str:
|
||||
file_path = Path(source.file_path)
|
||||
upload_dir = get_settings().upload_dir
|
||||
|
||||
relative_path: Path
|
||||
try:
|
||||
relative_path = file_path.resolve().relative_to(upload_dir.resolve())
|
||||
except ValueError:
|
||||
parts = file_path.parts
|
||||
if "uploads" in parts:
|
||||
uploads_index = parts.index("uploads")
|
||||
relative_path = Path(*parts[uploads_index + 1 :])
|
||||
else:
|
||||
relative_path = Path(file_path.name)
|
||||
|
||||
encoded_relative_path = "/".join(quote(part) for part in relative_path.parts)
|
||||
return f"{UPLOADS_URL_PREFIX}/{encoded_relative_path}"
|
||||
|
||||
|
||||
def _document_kind(source: Source) -> str:
|
||||
suffix = Path(source.file_path).suffix.lower()
|
||||
if suffix == ".pdf":
|
||||
return "pdf"
|
||||
return "image"
|
||||
|
||||
|
||||
def _attach_panzoom(host_id: str) -> None:
|
||||
ui.run_javascript(
|
||||
f"""
|
||||
(function() {{
|
||||
if (!window.Panzoom) return;
|
||||
window.__transcriptionPanzoom = window.__transcriptionPanzoom || {{}};
|
||||
const host = document.getElementById({host_id!r});
|
||||
if (!host) return;
|
||||
const target = host.querySelector('[data-panzoom-target]');
|
||||
const media = host.querySelector('[data-panzoom-media]');
|
||||
if (!target) return;
|
||||
|
||||
const cleanup = () => {{
|
||||
const existing = window.__transcriptionPanzoom[{host_id!r}];
|
||||
if (existing?.resizeObserver) existing.resizeObserver.disconnect();
|
||||
if (existing?.wheelHandler) host.removeEventListener('wheel', existing.wheelHandler);
|
||||
if (existing?.instance) existing.instance.destroy();
|
||||
}};
|
||||
|
||||
const computeFitScale = () => {{
|
||||
const hostRect = host.getBoundingClientRect();
|
||||
if (hostRect.width <= 0 || hostRect.height <= 0) return null;
|
||||
return 1;
|
||||
}};
|
||||
|
||||
const buildInstance = () => {{
|
||||
cleanup();
|
||||
|
||||
const fitScale = computeFitScale();
|
||||
if (fitScale === null) return false;
|
||||
|
||||
const minScale = Math.min(fitScale, 0.01);
|
||||
const instance = Panzoom(target, {{
|
||||
startX: 0,
|
||||
startY: 0,
|
||||
startScale: fitScale,
|
||||
minScale: minScale,
|
||||
maxScale: 256,
|
||||
step: 0.2,
|
||||
roundPixels: false,
|
||||
panOnlyWhenZoomed: true,
|
||||
overflow: 'hidden',
|
||||
}});
|
||||
|
||||
const wheelHandler = (event) => instance.zoomWithWheel(event);
|
||||
host.addEventListener('wheel', wheelHandler, {{ passive: false }});
|
||||
|
||||
requestAnimationFrame(() => {{
|
||||
instance.reset({{ animate: false }});
|
||||
}});
|
||||
|
||||
const resizeObserver = new ResizeObserver(() => {{
|
||||
const nextFitScale = computeFitScale();
|
||||
if (nextFitScale === null) return;
|
||||
instance.setOptions({{
|
||||
startScale: nextFitScale,
|
||||
minScale: Math.min(nextFitScale, 0.01),
|
||||
}});
|
||||
instance.reset({{ animate: false }});
|
||||
}});
|
||||
resizeObserver.observe(host);
|
||||
|
||||
window.__transcriptionPanzoom[{host_id!r}] = {{
|
||||
instance,
|
||||
wheelHandler,
|
||||
resizeObserver,
|
||||
}};
|
||||
return true;
|
||||
}};
|
||||
|
||||
const initWhenReady = (retries = 15) => {{
|
||||
if (buildInstance()) return;
|
||||
if (retries <= 0) return;
|
||||
requestAnimationFrame(() => initWhenReady(retries - 1));
|
||||
}};
|
||||
|
||||
if (media && media.tagName === 'IMG' && !media.complete) {{
|
||||
media.addEventListener('load', () => initWhenReady(), {{ once: true }});
|
||||
return;
|
||||
}}
|
||||
|
||||
initWhenReady();
|
||||
}})();
|
||||
"""
|
||||
)
|
||||
@@ -2,12 +2,39 @@
|
||||
|
||||
import re
|
||||
from datetime import date
|
||||
from uuid import UUID
|
||||
|
||||
from transcription.db.models import Person
|
||||
|
||||
YEAR_PATTERN = re.compile(r"\b[12]\d{3}\b")
|
||||
|
||||
|
||||
def parse_uuid(value: object | None) -> UUID | None:
|
||||
"""Parse a user-supplied identifier, treating anything unusable as absent."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, UUID):
|
||||
return value
|
||||
candidate = str(value).strip()
|
||||
if not candidate:
|
||||
return None
|
||||
try:
|
||||
return UUID(candidate)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_iso_date(value: str | None) -> date | None:
|
||||
"""Parse an ISO date from a form field, treating anything unusable as absent."""
|
||||
candidate = (value or "").strip()
|
||||
if not candidate:
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(candidate)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def compact_date(exact: date | None, approximate: str | None) -> str:
|
||||
"""Prefer an exact date, then an approximate value, then an unknown marker."""
|
||||
if exact is not None:
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Page-entry guards that render a terminal message instead of a record view."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from nicegui import ui
|
||||
|
||||
from transcription.ui.components.formatters import parse_uuid
|
||||
|
||||
_GUARD_CLASSES = "text-h6 ui-text-danger p-4"
|
||||
|
||||
|
||||
def render_guard_message(message: str) -> None:
|
||||
"""Render the single terminal message shown when a record cannot be displayed."""
|
||||
ui.label(message).classes(_GUARD_CLASSES)
|
||||
|
||||
|
||||
def parsed_record_id(value: str | None, *, noun: str) -> UUID | None:
|
||||
"""Parse a route identifier, rendering the invalid-id message when it is unusable.
|
||||
|
||||
`noun` is the capitalized record name, for example "Document".
|
||||
"""
|
||||
parsed = parse_uuid(value)
|
||||
if parsed is None:
|
||||
render_guard_message(f"Invalid {noun.lower()} id")
|
||||
return parsed
|
||||
|
||||
|
||||
def render_record_not_found(noun: str) -> None:
|
||||
"""Render the not-found message for a record that failed to load."""
|
||||
render_guard_message(f"{noun} not found")
|
||||
@@ -12,8 +12,10 @@ from transcription.db.models import DocumentPerson
|
||||
from transcription.db.models import Person
|
||||
from transcription.db.models import PersonRole
|
||||
from transcription.services.people import DocumentPersonInput
|
||||
from transcription.ui.components.formatters import parse_uuid
|
||||
from transcription.ui.components.formatters import person_selector_label
|
||||
from transcription.ui.components.primitives import destructive_button
|
||||
from transcription.ui.components.table.common import build_table
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -48,7 +50,7 @@ class LinkedPeopleEditor:
|
||||
"""Return the complete staged link set for persistence."""
|
||||
return [DocumentPersonInput(person_id=link.person_id, role_id=link.role_id) for link in self.links]
|
||||
|
||||
@ui.refreshable
|
||||
@ui.refreshable_method
|
||||
def render(self) -> None:
|
||||
ui.label("Linked People").classes("text-sm font-semibold ui-text-primary mt-2")
|
||||
ui.button("Create new person", on_click=lambda: ui.navigate.to("/people/new"), icon="person_add").props(
|
||||
@@ -63,16 +65,18 @@ class LinkedPeopleEditor:
|
||||
}
|
||||
for link in sorted(self.links, key=lambda item: self._person_label(item.person_id).casefold())
|
||||
]
|
||||
self.table = ui.table(
|
||||
columns=[
|
||||
self.table = build_table(
|
||||
rows,
|
||||
[
|
||||
{"name": "person", "label": "Person", "field": "person", "align": "left", "sortable": True},
|
||||
{"name": "role", "label": "Role", "field": "role", "align": "left", "sortable": True},
|
||||
],
|
||||
rows=rows,
|
||||
row_key="person_id",
|
||||
default_sort_by="person",
|
||||
show_search=False,
|
||||
selection="multiple",
|
||||
pagination={"rowsPerPage": 0, "sortBy": "person"},
|
||||
).classes("w-full ui-table")
|
||||
rows_per_page=0,
|
||||
row_key="person_id",
|
||||
)
|
||||
|
||||
with ui.row().classes("w-full items-center gap-2"):
|
||||
ui.button("Add", icon="add", on_click=self._begin_add).classes("ui-btn-primary")
|
||||
@@ -106,8 +110,8 @@ class LinkedPeopleEditor:
|
||||
role_input.value = str(current.role_id)
|
||||
|
||||
def save() -> None:
|
||||
person_id = self._parse_uuid(person_input.value)
|
||||
role_id = self._parse_uuid(role_input.value)
|
||||
person_id = parse_uuid(person_input.value)
|
||||
role_id = parse_uuid(role_input.value)
|
||||
if person_id is None or role_id is None:
|
||||
ui.notify("Select both a Person and Person Role.", type="warning")
|
||||
return
|
||||
@@ -171,10 +175,3 @@ class LinkedPeopleEditor:
|
||||
@staticmethod
|
||||
def _role_option_label(role: PersonRole) -> str:
|
||||
return role.label if role.is_active else f"{role.label} (inactive)"
|
||||
|
||||
@staticmethod
|
||||
def _parse_uuid(value: Any) -> UUID | None:
|
||||
try:
|
||||
return UUID(str(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Pure resolution of stored media paths into browser-reachable upload URLs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
_ABSOLUTE_SCHEMES = ("http://", "https://", "data:")
|
||||
_UPLOAD_ROUTE_PREFIX = "/uploads/"
|
||||
|
||||
|
||||
def absolute_upload_url(path: str, *, base_url: str) -> str:
|
||||
"""Join an application-relative upload path onto the request base URL."""
|
||||
base = base_url.rstrip("/")
|
||||
normalized_path = path if path.startswith("/") else f"/{path}"
|
||||
return f"{base}{normalized_path}"
|
||||
|
||||
|
||||
def resolve_media_url(path: str | None, *, upload_dir: Path, base_url: str) -> str | None:
|
||||
"""Map a stored media path onto a served upload URL.
|
||||
|
||||
Stored paths have accumulated several shapes over the life of the schema:
|
||||
absolute filesystem paths, paths relative to the working directory, paths
|
||||
relative to the upload root, and paths that already carry an upload route.
|
||||
All of them must still resolve, so each shape is tried in turn.
|
||||
"""
|
||||
candidate = (path or "").strip()
|
||||
if not candidate:
|
||||
return None
|
||||
|
||||
normalized = candidate.replace("\\", "/")
|
||||
lowered = normalized.casefold()
|
||||
if lowered.startswith(_ABSOLUTE_SCHEMES):
|
||||
return normalized
|
||||
if normalized.startswith(_UPLOAD_ROUTE_PREFIX):
|
||||
return absolute_upload_url(normalized, base_url=base_url)
|
||||
|
||||
resolved_upload_dir = upload_dir.resolve()
|
||||
path_obj = Path(candidate)
|
||||
|
||||
if path_obj.is_absolute():
|
||||
absolute_candidates = [path_obj.resolve()]
|
||||
else:
|
||||
absolute_candidates = [
|
||||
(Path.cwd() / path_obj).resolve(),
|
||||
(resolved_upload_dir / path_obj).resolve(),
|
||||
]
|
||||
|
||||
for absolute_candidate in absolute_candidates:
|
||||
try:
|
||||
relative = absolute_candidate.relative_to(resolved_upload_dir).as_posix()
|
||||
except ValueError:
|
||||
continue
|
||||
return absolute_upload_url(f"/uploads/{quote(relative)}", base_url=base_url)
|
||||
|
||||
upload_name = resolved_upload_dir.name.casefold()
|
||||
normalized_parts = Path(normalized).parts
|
||||
lowered_parts = [part.casefold() for part in normalized_parts]
|
||||
if upload_name in lowered_parts:
|
||||
index = lowered_parts.index(upload_name)
|
||||
relative = Path(*normalized_parts[index + 1 :]).as_posix()
|
||||
if relative:
|
||||
return absolute_upload_url(f"/uploads/{quote(relative)}", base_url=base_url)
|
||||
|
||||
if lowered.startswith("uploads/"):
|
||||
return absolute_upload_url(f"/{normalized}", base_url=base_url)
|
||||
if lowered.startswith("data/"):
|
||||
relative = normalized.split("/", 1)[1] if "/" in normalized else ""
|
||||
if relative:
|
||||
return absolute_upload_url(f"/uploads/{quote(relative)}", base_url=base_url)
|
||||
if lowered.startswith(("documents/", "persons/")):
|
||||
return absolute_upload_url(f"/uploads/{quote(normalized)}", base_url=base_url)
|
||||
|
||||
return absolute_upload_url(f"/uploads/{quote(path_obj.name)}", base_url=base_url)
|
||||
@@ -48,9 +48,12 @@ def build_table(
|
||||
show_search: bool = True,
|
||||
search_placeholder: str = "Search records...",
|
||||
on_row_click_id: Callable[[str], None] | None = None,
|
||||
selection: str | None = None,
|
||||
rows_per_page: int = 25,
|
||||
row_key: str = "id",
|
||||
) -> Any:
|
||||
"""Build a styled Quasar table widget with optional client-side filtering and row-click handlers."""
|
||||
pagination: dict[str, Any] = {"rowsPerPage": 25}
|
||||
pagination: dict[str, Any] = {"rowsPerPage": rows_per_page}
|
||||
if default_sort_by is not None:
|
||||
pagination["sortBy"] = default_sort_by
|
||||
pagination["descending"] = default_descending
|
||||
@@ -65,13 +68,17 @@ def build_table(
|
||||
.classes("w-64 text-xs ui-form-surface")
|
||||
)
|
||||
|
||||
table_kwargs: dict[str, Any] = {
|
||||
"rows": rows,
|
||||
"columns": columns,
|
||||
"row_key": row_key,
|
||||
"pagination": pagination,
|
||||
}
|
||||
if selection is not None:
|
||||
table_kwargs["selection"] = selection
|
||||
|
||||
table = (
|
||||
ui.table(
|
||||
rows=rows,
|
||||
columns=columns,
|
||||
row_key="id",
|
||||
pagination=pagination,
|
||||
)
|
||||
ui.table(**table_kwargs)
|
||||
.classes(f"w-full ui-table {classes}".strip())
|
||||
.props(
|
||||
'flat square binary-state-sort table-style="table-layout: fixed; width: 100%;" '
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Table rendering for the label registries edited on the settings page."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from transcription.ui.components.table.common import build_table
|
||||
|
||||
_ACTIVE_CELL = """
|
||||
<q-td :props="props" class="text-center">
|
||||
<q-checkbox :model-value="props.value" disable dense />
|
||||
</q-td>
|
||||
"""
|
||||
_BUILT_IN_CELL = """
|
||||
<q-td :props="props" class="text-center">
|
||||
{{ props.value ? 'Yes' : 'No' }}
|
||||
</q-td>
|
||||
"""
|
||||
|
||||
|
||||
def render_registry_table(
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
count_field: str,
|
||||
count_label: str,
|
||||
) -> Any:
|
||||
"""Render one label registry with its usage count and read-only status flags."""
|
||||
table = build_table(
|
||||
rows,
|
||||
[
|
||||
{"name": "label", "label": "Label", "field": "label", "align": "left", "sortable": True},
|
||||
{
|
||||
"name": count_field,
|
||||
"label": count_label,
|
||||
"field": count_field,
|
||||
"align": "right",
|
||||
"sortable": True,
|
||||
},
|
||||
{"name": "is_active", "label": "Active", "field": "is_active", "align": "center"},
|
||||
{"name": "is_built_in", "label": "Built-in", "field": "is_built_in", "align": "center"},
|
||||
],
|
||||
default_sort_by="label",
|
||||
show_search=False,
|
||||
selection="single",
|
||||
rows_per_page=0,
|
||||
)
|
||||
table.add_slot("body-cell-is_active", _ACTIVE_CELL)
|
||||
table.add_slot("body-cell-is_built_in", _BUILT_IN_CELL)
|
||||
return table
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Shared file-picker wiring for the upload surfaces."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
from nicegui import ui
|
||||
|
||||
from transcription.services.source_media import SOURCE_EXTENSIONS
|
||||
|
||||
SOURCE_UPLOAD_EXTENSIONS: tuple[str, ...] = tuple(sorted(SOURCE_EXTENSIONS))
|
||||
IMAGE_UPLOAD_EXTENSIONS: tuple[str, ...] = (
|
||||
".bmp",
|
||||
".gif",
|
||||
".jpeg",
|
||||
".jpg",
|
||||
".png",
|
||||
".tif",
|
||||
".tiff",
|
||||
".webp",
|
||||
)
|
||||
|
||||
|
||||
def accept_attribute(extensions: Iterable[str]) -> str:
|
||||
"""Build the HTML ``accept`` attribute value for a set of extensions."""
|
||||
return f'accept="{",".join(extensions)}"'
|
||||
|
||||
|
||||
def render_upload_picker(
|
||||
*,
|
||||
on_upload: Callable[[Any], Awaitable[None]],
|
||||
label: str,
|
||||
extensions: Iterable[str],
|
||||
directory: bool = False,
|
||||
multiple: bool = False,
|
||||
) -> ui.upload:
|
||||
"""Render the standard auto-uploading file picker used across pages."""
|
||||
props = [accept_attribute(extensions)]
|
||||
if directory:
|
||||
props.append("webkitdirectory directory")
|
||||
if multiple:
|
||||
props.append("multiple")
|
||||
return (
|
||||
ui.upload(on_upload=on_upload, auto_upload=True, label=label)
|
||||
.props(" ".join(props))
|
||||
.classes("w-full")
|
||||
)
|
||||
@@ -1,62 +1,97 @@
|
||||
"""File-backed storage helpers for the homepage content."""
|
||||
"""File-backed storage helpers for the homepage content.
|
||||
|
||||
The homepage storage root is a configured setting (``homepage_dir``) like every
|
||||
other storage root, rather than a path derived from this module's location. The
|
||||
previous ``Path(__file__).parents[3]`` form was both unconfigurable and wrong
|
||||
outside a source checkout, since an installed distribution would resolve it into
|
||||
the package directory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
HOME_PAGE_DIR = Path(__file__).resolve().parents[3] / "data" / "homepage"
|
||||
HOME_PAGE_MARKDOWN_PATH = HOME_PAGE_DIR / "homepage.md"
|
||||
from transcription.config import Settings
|
||||
from transcription.config import get_settings
|
||||
from transcription.errors import AppError
|
||||
from transcription.services.media_storage import write_media_bytes
|
||||
|
||||
HOME_PAGE_MARKDOWN_NAME = "homepage.md"
|
||||
SUPPORTED_IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tif", ".tiff"}
|
||||
|
||||
|
||||
def ensure_homepage_storage() -> None:
|
||||
"""Create the homepage storage directory when needed."""
|
||||
HOME_PAGE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
class HomepageStorageError(AppError):
|
||||
"""Raised when homepage media cannot be persisted."""
|
||||
|
||||
|
||||
def read_homepage_markdown() -> str:
|
||||
def homepage_dir(settings: Settings | None = None) -> Path:
|
||||
"""Return the configured homepage storage directory."""
|
||||
return (settings or get_settings()).homepage_dir
|
||||
|
||||
|
||||
def homepage_markdown_path(settings: Settings | None = None) -> Path:
|
||||
"""Return the configured homepage markdown file path."""
|
||||
return homepage_dir(settings) / HOME_PAGE_MARKDOWN_NAME
|
||||
|
||||
|
||||
def ensure_homepage_storage(settings: Settings | None = None) -> Path:
|
||||
"""Create the homepage storage directory when needed and return it."""
|
||||
directory = homepage_dir(settings)
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
return directory
|
||||
|
||||
|
||||
def read_homepage_markdown(settings: Settings | None = None) -> str:
|
||||
"""Read the saved homepage markdown text."""
|
||||
ensure_homepage_storage()
|
||||
if not HOME_PAGE_MARKDOWN_PATH.exists():
|
||||
ensure_homepage_storage(settings)
|
||||
path = homepage_markdown_path(settings)
|
||||
if not path.exists():
|
||||
return ""
|
||||
return HOME_PAGE_MARKDOWN_PATH.read_text(encoding="utf-8")
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def save_homepage_markdown(markdown_text: str) -> None:
|
||||
def save_homepage_markdown(markdown_text: str, settings: Settings | None = None) -> None:
|
||||
"""Persist the homepage markdown text."""
|
||||
ensure_homepage_storage()
|
||||
HOME_PAGE_MARKDOWN_PATH.write_text(markdown_text, encoding="utf-8")
|
||||
ensure_homepage_storage(settings)
|
||||
homepage_markdown_path(settings).write_text(markdown_text, encoding="utf-8")
|
||||
|
||||
|
||||
def store_homepage_image(*, filename: str, file_bytes: bytes) -> Path:
|
||||
async def store_homepage_image(
|
||||
*,
|
||||
filename: str,
|
||||
file_bytes: bytes,
|
||||
settings: Settings | None = None,
|
||||
) -> Path:
|
||||
"""Persist an uploaded homepage image in the shared homepage folder."""
|
||||
ensure_homepage_storage()
|
||||
|
||||
safe_name = Path(filename).name
|
||||
if not safe_name:
|
||||
msg = "Homepage image filename is required"
|
||||
raise ValueError(msg)
|
||||
|
||||
stored_path = HOME_PAGE_DIR / safe_name
|
||||
stored_path.write_bytes(file_bytes)
|
||||
return stored_path
|
||||
return await write_media_bytes(
|
||||
target_dir=homepage_dir(settings),
|
||||
stored_name=safe_name,
|
||||
file_bytes=file_bytes,
|
||||
error=HomepageStorageError,
|
||||
failure_message="Failed to persist homepage image",
|
||||
failure_suggestion="Check homepage directory permissions and available disk space, then retry.",
|
||||
log_label="homepage image",
|
||||
)
|
||||
|
||||
|
||||
def list_homepage_images() -> list[Path]:
|
||||
def list_homepage_images(settings: Settings | None = None) -> list[Path]:
|
||||
"""List stored homepage images in the order they were last updated."""
|
||||
ensure_homepage_storage()
|
||||
directory = ensure_homepage_storage(settings)
|
||||
|
||||
image_paths = [
|
||||
path
|
||||
for path in HOME_PAGE_DIR.iterdir()
|
||||
if path.is_file() and path.suffix.lower() in SUPPORTED_IMAGE_SUFFIXES
|
||||
path for path in directory.iterdir() if path.is_file() and path.suffix.lower() in SUPPORTED_IMAGE_SUFFIXES
|
||||
]
|
||||
return sorted(image_paths, key=lambda path: (path.stat().st_mtime, path.name))
|
||||
|
||||
|
||||
def latest_homepage_image() -> Path | None:
|
||||
def latest_homepage_image(settings: Settings | None = None) -> Path | None:
|
||||
"""Return the most recently updated homepage image, if one exists."""
|
||||
image_paths = list_homepage_images()
|
||||
image_paths = list_homepage_images(settings)
|
||||
if not image_paths:
|
||||
return None
|
||||
return image_paths[-1]
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
@@ -21,10 +21,17 @@ from transcription.services.workflows import create_document_with_people
|
||||
from transcription.services.workflows import update_document_with_people
|
||||
from transcription.ui.components.app_shell import render_navigation_header
|
||||
from transcription.ui.components.cards import archival_card
|
||||
from transcription.ui.components.confirm_delete import dependency_summary
|
||||
from transcription.ui.components.confirm_delete import render_delete_actions
|
||||
from transcription.ui.components.confirm_delete import render_delete_blocked_notice
|
||||
from transcription.ui.components.data_display import archival_badge
|
||||
from transcription.ui.components.data_display import metadata_row
|
||||
from transcription.ui.components.error_presenter import show_error
|
||||
from transcription.ui.components.formatters import compact_date
|
||||
from transcription.ui.components.formatters import parse_iso_date
|
||||
from transcription.ui.components.formatters import parse_uuid
|
||||
from transcription.ui.components.guards import parsed_record_id
|
||||
from transcription.ui.components.guards import render_record_not_found
|
||||
from transcription.ui.components.linked_people import LinkedPeopleEditor
|
||||
from transcription.ui.components.linked_people import StagedLinkedPerson
|
||||
from transcription.ui.components.primitives import destructive_button
|
||||
@@ -38,6 +45,20 @@ from transcription.ui.theme import page_header
|
||||
from ...db.session import SessionFactoryDep
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DocumentFormFields:
|
||||
"""Bound input widgets for the Document create and edit forms."""
|
||||
|
||||
name: ui.input
|
||||
document_type: ui.select
|
||||
type_options: dict[str, str]
|
||||
document_date: ui.input
|
||||
document_date_raw: ui.input
|
||||
location: ui.input
|
||||
archive: ui.input
|
||||
notes: ui.textarea
|
||||
|
||||
|
||||
def register_page() -> None: # noqa: PLR0915
|
||||
"""Register documents list and detail routes."""
|
||||
|
||||
@@ -53,7 +74,7 @@ def register_page() -> None: # noqa: PLR0915
|
||||
people = sorted(await people_service.list_people(), key=lambda item: item.full_name.casefold())
|
||||
role_catalog = list(await people_service.list_person_roles(active_only=False))
|
||||
type_catalog = await document_service.list_document_types()
|
||||
requested_person_id = _parse_uuid(request.query_params.get("person_id"))
|
||||
requested_person_id = parse_uuid(request.query_params.get("person_id"))
|
||||
staged_links: list[StagedLinkedPerson] = []
|
||||
if requested_person_id is not None and any(person.id == requested_person_id for person in people):
|
||||
try:
|
||||
@@ -82,8 +103,8 @@ def register_page() -> None: # noqa: PLR0915
|
||||
return_to = request.query_params.get("return_to")
|
||||
|
||||
async def submit_create() -> None:
|
||||
candidate_name = (form["name"].value or "").strip()
|
||||
candidate_type_id = _resolve_selected_document_type_id(form["type"].value, form["type_options"])
|
||||
candidate_name = (form.name.value or "").strip()
|
||||
candidate_type_id = _resolve_selected_document_type_id(form.document_type.value, form.type_options)
|
||||
if not candidate_name:
|
||||
ui.notify("Document name is required.", type="warning")
|
||||
return
|
||||
@@ -91,8 +112,8 @@ def register_page() -> None: # noqa: PLR0915
|
||||
ui.notify("Document type is required.", type="warning")
|
||||
return
|
||||
|
||||
parsed_date = _parse_iso_date(form["date"].value)
|
||||
if form["date"].value and parsed_date is None:
|
||||
parsed_date = parse_iso_date(form.document_date.value)
|
||||
if form.document_date.value and parsed_date is None:
|
||||
ui.notify("Exact date must use YYYY-MM-DD.", type="warning")
|
||||
return
|
||||
|
||||
@@ -100,10 +121,10 @@ def register_page() -> None: # noqa: PLR0915
|
||||
name=candidate_name,
|
||||
document_type_id=candidate_type_id,
|
||||
document_date=parsed_date,
|
||||
document_date_raw=(form["date_raw"].value or "").strip() or None,
|
||||
location_created=(form["location"].value or "").strip() or None,
|
||||
notes=(form["notes"].value or "").strip() or None,
|
||||
archive_identifier=(form["archive"].value or "").strip() or None,
|
||||
document_date_raw=(form.document_date_raw.value or "").strip() or None,
|
||||
location_created=(form.location.value or "").strip() or None,
|
||||
notes=(form.notes.value or "").strip() or None,
|
||||
archive_identifier=(form.archive.value or "").strip() or None,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -169,15 +190,14 @@ def register_page() -> None: # noqa: PLR0915
|
||||
document_service = DocumentService(session_factory=session_factory)
|
||||
render_navigation_header(current_path="/documents")
|
||||
|
||||
parsed_doc_id = _parse_uuid(document_id)
|
||||
parsed_doc_id = parsed_record_id(document_id, noun="Document")
|
||||
if parsed_doc_id is None:
|
||||
ui.label("Invalid document id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
document = await document_service.read_document_detail(document_id=parsed_doc_id)
|
||||
except DocumentError:
|
||||
ui.label("Document not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Document")
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
show_error(exc, title="Load failed", operation="documents.read")
|
||||
@@ -216,15 +236,14 @@ def register_page() -> None: # noqa: PLR0915
|
||||
document_service = DocumentService(session_factory=session_factory)
|
||||
render_navigation_header(current_path="/documents")
|
||||
|
||||
parsed_doc_id = _parse_uuid(document_id)
|
||||
parsed_doc_id = parsed_record_id(document_id, noun="Document")
|
||||
if parsed_doc_id is None:
|
||||
ui.label("Invalid document id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
document = await document_service.read_document_detail(document_id=parsed_doc_id)
|
||||
except DocumentError:
|
||||
ui.label("Document not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Document")
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
show_error(exc, title="Load failed", operation="documents.jobs")
|
||||
@@ -272,15 +291,14 @@ def register_page() -> None: # noqa: PLR0915
|
||||
people_service = PeopleService(session_factory=session_factory)
|
||||
render_navigation_header(current_path="/documents")
|
||||
|
||||
parsed_doc_id = _parse_uuid(document_id)
|
||||
parsed_doc_id = parsed_record_id(document_id, noun="Document")
|
||||
if parsed_doc_id is None:
|
||||
ui.label("Invalid document id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
document = await document_service.read_document_detail(document_id=parsed_doc_id)
|
||||
except DocumentError:
|
||||
ui.label("Document not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Document")
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
show_error(exc, title="Load failed", operation="documents.edit.read")
|
||||
@@ -304,8 +322,8 @@ def register_page() -> None: # noqa: PLR0915
|
||||
)
|
||||
|
||||
async def submit_edit() -> None:
|
||||
candidate_name = (form["name"].value or "").strip()
|
||||
candidate_type_id = _resolve_selected_document_type_id(form["type"].value, form["type_options"])
|
||||
candidate_name = (form.name.value or "").strip()
|
||||
candidate_type_id = _resolve_selected_document_type_id(form.document_type.value, form.type_options)
|
||||
if not candidate_name:
|
||||
ui.notify("Document name is required.", type="warning")
|
||||
return
|
||||
@@ -313,8 +331,8 @@ def register_page() -> None: # noqa: PLR0915
|
||||
ui.notify("Document type is required.", type="warning")
|
||||
return
|
||||
|
||||
parsed_date = _parse_iso_date(form["date"].value)
|
||||
if form["date"].value and parsed_date is None:
|
||||
parsed_date = parse_iso_date(form.document_date.value)
|
||||
if form.document_date.value and parsed_date is None:
|
||||
ui.notify("Exact date must use YYYY-MM-DD.", type="warning")
|
||||
return
|
||||
|
||||
@@ -323,10 +341,10 @@ def register_page() -> None: # noqa: PLR0915
|
||||
name=candidate_name,
|
||||
document_type_id=candidate_type_id,
|
||||
document_date=parsed_date,
|
||||
document_date_raw=(form["date_raw"].value or "").strip() or None,
|
||||
location_created=(form["location"].value or "").strip() or None,
|
||||
notes=(form["notes"].value or "").strip() or None,
|
||||
archive_identifier=(form["archive"].value or "").strip() or None,
|
||||
document_date_raw=(form.document_date_raw.value or "").strip() or None,
|
||||
location_created=(form.location.value or "").strip() or None,
|
||||
notes=(form.notes.value or "").strip() or None,
|
||||
archive_identifier=(form.archive.value or "").strip() or None,
|
||||
created_at=document.created_at,
|
||||
updated_at=document.updated_at,
|
||||
)
|
||||
@@ -356,15 +374,14 @@ def register_page() -> None: # noqa: PLR0915
|
||||
document_service = DocumentService(session_factory=session_factory)
|
||||
render_navigation_header(current_path="/documents")
|
||||
|
||||
parsed_doc_id = _parse_uuid(document_id)
|
||||
parsed_doc_id = parsed_record_id(document_id, noun="Document")
|
||||
if parsed_doc_id is None:
|
||||
ui.label("Invalid document id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
document = await document_service.read_document_detail(document_id=parsed_doc_id)
|
||||
except DocumentError:
|
||||
ui.label("Document not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Document")
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
show_error(exc, title="Load failed", operation="documents.delete.read")
|
||||
@@ -377,27 +394,14 @@ def register_page() -> None: # noqa: PLR0915
|
||||
ui.label(f"Document: {document.name}").classes("text-sm font-semibold ui-text-primary")
|
||||
|
||||
if document.sources or document.jobs:
|
||||
ui.label("Delete is blocked because related records exist.").classes(
|
||||
"text-xs ui-text-danger font-bold mt-2"
|
||||
)
|
||||
deps = [
|
||||
cat
|
||||
for cat, present in [("Sources", bool(document.sources)), ("Jobs", bool(document.jobs))]
|
||||
if present
|
||||
]
|
||||
ui.label(f"Dependencies present: {', '.join(deps)}").classes("text-xs ui-text-muted")
|
||||
ui.label("Remove related records first, then retry deletion.").classes(
|
||||
"text-xs ui-text-muted italic"
|
||||
)
|
||||
|
||||
with ui.row().classes("w-full items-center gap-2 mt-4"):
|
||||
ui.button(
|
||||
"Back to Document",
|
||||
on_click=lambda: ui.navigate.to(f"/documents/{document.id}"),
|
||||
icon="arrow_back",
|
||||
).classes("ui-btn-primary text-xs")
|
||||
ui.button("Go to Jobs", on_click=lambda: ui.navigate.to("/jobs"), icon="work_history").props(
|
||||
"flat text-xs"
|
||||
render_delete_blocked_notice(
|
||||
reason="Delete is blocked because related records exist.",
|
||||
detail=dependency_summary(
|
||||
[("Sources", bool(document.sources)), ("Jobs", bool(document.jobs))]
|
||||
),
|
||||
guidance="Remove related records first, then retry deletion.",
|
||||
back_label="Back to Document",
|
||||
back_target=f"/documents/{document.id}",
|
||||
)
|
||||
return
|
||||
|
||||
@@ -424,13 +428,11 @@ def register_page() -> None: # noqa: PLR0915
|
||||
ui.notify("Document deleted", type="positive")
|
||||
ui.navigate.to("/documents")
|
||||
|
||||
with ui.row().classes("w-full items-center gap-2 mt-2"):
|
||||
destructive_button(
|
||||
"Delete document permanently", on_click=submit_delete, icon="delete_forever", variant="solid"
|
||||
render_delete_actions(
|
||||
confirm_label="Delete document permanently",
|
||||
on_confirm=submit_delete,
|
||||
cancel_target=f"/documents/{document.id}",
|
||||
)
|
||||
ui.button(
|
||||
"Cancel", on_click=lambda: ui.navigate.to(f"/documents/{document.id}"), icon="arrow_back"
|
||||
).props("flat")
|
||||
|
||||
|
||||
# --- Helper Sub-Components ---
|
||||
@@ -441,7 +443,7 @@ def _render_document_form_fields(
|
||||
document: Document | None = None,
|
||||
type_options: dict[str, str],
|
||||
linked_people: LinkedPeopleEditor,
|
||||
) -> dict[str, Any]:
|
||||
) -> DocumentFormFields:
|
||||
with archival_card(extra_classes="gap-3"):
|
||||
name_input = (
|
||||
ui.input(label="Document name", value=document.name if document else "")
|
||||
@@ -504,16 +506,16 @@ def _render_document_form_fields(
|
||||
|
||||
linked_people.render()
|
||||
|
||||
return {
|
||||
"name": name_input,
|
||||
"type": type_input,
|
||||
"type_options": type_display_to_id,
|
||||
"date": date_input,
|
||||
"date_raw": date_raw_input,
|
||||
"location": location_input,
|
||||
"archive": archive_input,
|
||||
"notes": notes_input,
|
||||
}
|
||||
return DocumentFormFields(
|
||||
name=name_input,
|
||||
document_type=type_input,
|
||||
type_options=type_display_to_id,
|
||||
document_date=date_input,
|
||||
document_date_raw=date_raw_input,
|
||||
location=location_input,
|
||||
archive=archive_input,
|
||||
notes=notes_input,
|
||||
)
|
||||
|
||||
|
||||
def _render_bento_viewer_zone(document: Document) -> None:
|
||||
@@ -588,31 +590,12 @@ def _render_document_processing_card(document: Document) -> None:
|
||||
).classes("ui-btn-primary text-xs")
|
||||
|
||||
|
||||
def _parse_uuid(value: str | None) -> UUID | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return UUID(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_iso_date(value: str | None) -> date | None:
|
||||
candidate = (value or "").strip()
|
||||
if not candidate:
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(candidate)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_selected_document_type_id(selected_value: Any, type_options: dict[str, str]) -> UUID | None:
|
||||
candidate = str(selected_value).strip() if selected_value is not None else ""
|
||||
if not candidate:
|
||||
return None
|
||||
selected_id = type_options.get(candidate)
|
||||
return _parse_uuid(selected_id)
|
||||
return parse_uuid(selected_id)
|
||||
|
||||
|
||||
def _group_people_by_role(document: Document) -> dict[str, list[Any]]:
|
||||
|
||||
@@ -2,12 +2,15 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from nicegui import events
|
||||
from nicegui import ui
|
||||
|
||||
from transcription.ui.components.app_shell import render_navigation_header
|
||||
from transcription.ui.components.cards import archival_card
|
||||
from transcription.ui.components.primitives import render_empty_state
|
||||
from transcription.ui.components.primitives import section_header_row
|
||||
from transcription.ui.components.upload_panel import IMAGE_UPLOAD_EXTENSIONS
|
||||
from transcription.ui.components.upload_panel import render_upload_picker
|
||||
from transcription.ui.components.viewers import dark_room_viewer
|
||||
from transcription.ui.homepage_store import latest_homepage_image
|
||||
from transcription.ui.homepage_store import read_homepage_markdown
|
||||
@@ -35,9 +38,11 @@ def _render_homepage_editor(*, render_image_panel, markdown_input, on_upload) ->
|
||||
with ui.grid().classes("w-full grid-cols-12 gap-4"):
|
||||
with ui.column().classes("col-span-12 lg:col-span-4 gap-4"):
|
||||
with archival_card(title="Homepage Image"):
|
||||
ui.upload(on_upload=on_upload, auto_upload=True, label="Upload image").props(
|
||||
'accept=".jpg,.jpeg,.png,.gif,.webp,.bmp,.tif,.tiff"'
|
||||
).classes("w-full")
|
||||
render_upload_picker(
|
||||
on_upload=on_upload,
|
||||
label="Upload image",
|
||||
extensions=IMAGE_UPLOAD_EXTENSIONS,
|
||||
)
|
||||
render_image_panel()
|
||||
|
||||
with ui.column().classes("col-span-12 lg:col-span-5 gap-4"), archival_card(title="Home Text"):
|
||||
@@ -82,9 +87,9 @@ def register_page() -> None:
|
||||
def render_image_panel() -> None:
|
||||
dark_room_viewer(str(preview_image[0]) if preview_image[0] else None, count_label="Homepage Image")
|
||||
|
||||
async def on_upload(event) -> None:
|
||||
async def on_upload(event: events.UploadEventArguments) -> None:
|
||||
payload = await event.file.read()
|
||||
preview_image[0] = store_homepage_image(filename=event.file.name, file_bytes=payload)
|
||||
preview_image[0] = await store_homepage_image(filename=event.file.name, file_bytes=payload)
|
||||
ui.notify(f"Uploaded {event.file.name}", type="positive")
|
||||
render_image_panel.refresh()
|
||||
|
||||
|
||||
@@ -7,14 +7,12 @@ from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import Request
|
||||
from nicegui import events
|
||||
from nicegui import ui
|
||||
|
||||
from transcription.config import Settings
|
||||
from transcription.config import get_settings
|
||||
from transcription.db.models import Job
|
||||
from transcription.db.models import JobSourceStatus
|
||||
from transcription.db.models import JobStatus
|
||||
from transcription.db.session import session_scope
|
||||
from transcription.services import ServiceBundle
|
||||
from transcription.services.documents import DocumentService
|
||||
from transcription.services.jobs import JobCancelBlockedError
|
||||
@@ -26,19 +24,29 @@ from transcription.services.store import create_job_for_document
|
||||
from transcription.services.workflows import create_source_retranscription_job
|
||||
from transcription.ui.components.app_shell import render_navigation_header
|
||||
from transcription.ui.components.cards import archival_card
|
||||
from transcription.ui.components.confirm_delete import render_delete_actions
|
||||
from transcription.ui.components.confirm_delete import render_delete_blocked_notice
|
||||
from transcription.ui.components.data_display import archival_badge
|
||||
from transcription.ui.components.data_display import metadata_row
|
||||
from transcription.ui.components.error_presenter import show_error
|
||||
from transcription.ui.components.formatters import parse_uuid
|
||||
from transcription.ui.components.guards import parsed_record_id
|
||||
from transcription.ui.components.guards import render_record_not_found
|
||||
from transcription.ui.components.primitives import destructive_button
|
||||
from transcription.ui.components.primitives import render_empty_state
|
||||
from transcription.ui.components.primitives import section_header_row
|
||||
from transcription.ui.components.table.jobs import JobTableRow
|
||||
from transcription.ui.components.table.jobs import render_jobs_table
|
||||
from transcription.ui.components.upload_panel import SOURCE_UPLOAD_EXTENSIONS
|
||||
from transcription.ui.components.upload_panel import render_upload_picker
|
||||
from transcription.ui.runtime import resolve_runtime_settings
|
||||
from transcription.ui.theme import page_header
|
||||
from transcription.worker import resolve_worker_notifier
|
||||
|
||||
from ...db.session import SessionFactoryDep
|
||||
|
||||
JOB_DETAIL_REFRESH_INTERVAL_SECONDS = 4.0
|
||||
|
||||
|
||||
def register_page() -> None: # noqa: PLR0915
|
||||
"""Register jobs list and detail routes."""
|
||||
@@ -81,7 +89,7 @@ def register_page() -> None: # noqa: PLR0915
|
||||
) -> None:
|
||||
documents_service = DocumentService(session_factory=session_factory)
|
||||
sources_service = SourceService(session_factory=session_factory)
|
||||
settings = _resolve_runtime_settings(request)
|
||||
settings = resolve_runtime_settings(request)
|
||||
render_navigation_header(current_path="/jobs")
|
||||
|
||||
with ui.column().classes("w-full max-w-4xl mx-auto p-4 gap-4"):
|
||||
@@ -89,7 +97,7 @@ def register_page() -> None: # noqa: PLR0915
|
||||
"Create Processing Job", subtitle="Queue source files for AI transcription and entity processing."
|
||||
)
|
||||
|
||||
requested_source_id = _parse_uuid(request.query_params.get("source_id"))
|
||||
requested_source_id = parse_uuid(request.query_params.get("source_id"))
|
||||
try:
|
||||
locked_source = (
|
||||
await sources_service.read_source_detail(requested_source_id)
|
||||
@@ -182,13 +190,12 @@ def register_page() -> None: # noqa: PLR0915
|
||||
return
|
||||
|
||||
try:
|
||||
async with session_scope(session_factory=session_factory) as session:
|
||||
result = await create_job_for_document(
|
||||
document_id=document_id,
|
||||
source_files=uploaded_files,
|
||||
provider=(provider_input.value or None),
|
||||
model=(model_input.value or None),
|
||||
session=session,
|
||||
session_factory=session_factory,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
show_error(exc, title="Create job failed", operation="jobs.create")
|
||||
@@ -209,15 +216,14 @@ def register_page() -> None: # noqa: PLR0915
|
||||
jobs_service = JobService(session_factory=session_factory)
|
||||
render_navigation_header(current_path="/jobs")
|
||||
|
||||
parsed_job_id = _parse_uuid(job_id)
|
||||
parsed_job_id = parsed_record_id(job_id, noun="Job")
|
||||
if parsed_job_id is None:
|
||||
ui.label("Invalid job id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
job = await jobs_service.read_job(job_id=parsed_job_id)
|
||||
except ValueError:
|
||||
ui.label("Job not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Job")
|
||||
return
|
||||
|
||||
with ui.column().classes("w-full max-w-4xl mx-auto p-4 gap-4"):
|
||||
@@ -238,34 +244,38 @@ def register_page() -> None: # noqa: PLR0915
|
||||
ui.label("This page updates automatically while the job is active.").classes("text-xs ui-text-muted")
|
||||
|
||||
async def refresh_job() -> None:
|
||||
def stop_refresh() -> None:
|
||||
timer = timer_holder[0]
|
||||
if timer is not None:
|
||||
timer.cancel()
|
||||
timer_holder[0] = None
|
||||
|
||||
try:
|
||||
current_job[0] = await jobs_service.read_job(job_id=parsed_job_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if timer_holder[0] is not None:
|
||||
timer_holder[0].active = False
|
||||
stop_refresh()
|
||||
show_error(exc, title="Auto-refresh failed", operation="jobs.detail.refresh")
|
||||
return
|
||||
|
||||
render_detail.refresh()
|
||||
if current_job[0].status not in {JobStatus.QUEUED, JobStatus.PROCESSING}:
|
||||
timer_holder[0].active = False
|
||||
stop_refresh()
|
||||
|
||||
timer_holder[0] = ui.timer(4.0, refresh_job)
|
||||
timer_holder[0] = ui.timer(JOB_DETAIL_REFRESH_INTERVAL_SECONDS, refresh_job)
|
||||
|
||||
@ui.page("/jobs/{job_id}/cancel")
|
||||
async def job_cancel_page(job_id: str, request: Request, session_factory: SessionFactoryDep) -> None:
|
||||
jobs_service = JobService(session_factory=session_factory)
|
||||
render_navigation_header(current_path="/jobs")
|
||||
|
||||
parsed_job_id = _parse_uuid(job_id)
|
||||
parsed_job_id = parsed_record_id(job_id, noun="Job")
|
||||
if parsed_job_id is None:
|
||||
ui.label("Invalid job id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
job = await jobs_service.read_job(job_id=parsed_job_id)
|
||||
except ValueError:
|
||||
ui.label("Job not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Job")
|
||||
return
|
||||
|
||||
with ui.column().classes("w-full max-w-xl mx-auto p-4 gap-4"):
|
||||
@@ -307,15 +317,14 @@ def register_page() -> None: # noqa: PLR0915
|
||||
jobs_service = JobService(session_factory=session_factory)
|
||||
render_navigation_header(current_path="/jobs")
|
||||
|
||||
parsed_job_id = _parse_uuid(job_id)
|
||||
parsed_job_id = parsed_record_id(job_id, noun="Job")
|
||||
if parsed_job_id is None:
|
||||
ui.label("Invalid job id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
job = await jobs_service.read_job(job_id=parsed_job_id)
|
||||
except ValueError:
|
||||
ui.label("Job not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Job")
|
||||
return
|
||||
|
||||
failed_count = sum(1 for js in job.job_sources if js.status == JobSourceStatus.FAILED)
|
||||
@@ -360,15 +369,14 @@ def register_page() -> None: # noqa: PLR0915
|
||||
jobs_service = JobService(session_factory=session_factory)
|
||||
render_navigation_header(current_path="/jobs")
|
||||
|
||||
parsed_job_id = _parse_uuid(job_id)
|
||||
parsed_job_id = parsed_record_id(job_id, noun="Job")
|
||||
if parsed_job_id is None:
|
||||
ui.label("Invalid job id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
job = await jobs_service.read_job(job_id=parsed_job_id)
|
||||
except ValueError:
|
||||
ui.label("Job not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Job")
|
||||
return
|
||||
|
||||
with ui.column().classes("w-full max-w-xl mx-auto p-4 gap-4"):
|
||||
@@ -378,18 +386,12 @@ def register_page() -> None: # noqa: PLR0915
|
||||
ui.label(f"Job ID: {job.id}").classes("text-sm font-semibold font-mono ui-text-primary")
|
||||
|
||||
if job.status == JobStatus.PROCESSING:
|
||||
ui.label("Delete is blocked while the job is processing.").classes(
|
||||
"text-xs ui-text-danger font-bold mt-2"
|
||||
)
|
||||
ui.label("Wait for processing to complete, then retry delete.").classes(
|
||||
"text-xs ui-text-muted italic"
|
||||
)
|
||||
with ui.row().classes("w-full items-center gap-2 mt-4"):
|
||||
ui.button(
|
||||
"Back to Job", on_click=lambda: ui.navigate.to(f"/jobs/{job.id}"), icon="arrow_back"
|
||||
).classes("ui-btn-primary text-xs")
|
||||
ui.button("Back to Jobs", on_click=lambda: ui.navigate.to("/jobs"), icon="work_history").props(
|
||||
"flat text-xs"
|
||||
render_delete_blocked_notice(
|
||||
reason="Delete is blocked while the job is processing.",
|
||||
guidance="Wait for processing to complete, then retry delete.",
|
||||
back_label="Back to Job",
|
||||
back_target=f"/jobs/{job.id}",
|
||||
secondary_label="Back to Jobs",
|
||||
)
|
||||
return
|
||||
|
||||
@@ -421,11 +423,11 @@ def register_page() -> None: # noqa: PLR0915
|
||||
ui.notify("Job deleted", type="positive")
|
||||
ui.navigate.to("/jobs")
|
||||
|
||||
with ui.row().classes("w-full items-center gap-2 mt-2"):
|
||||
destructive_button(
|
||||
"Delete job and evidence", on_click=submit_delete, icon="delete_forever", variant="solid"
|
||||
render_delete_actions(
|
||||
confirm_label="Delete job and evidence",
|
||||
on_confirm=submit_delete,
|
||||
cancel_target=f"/jobs/{job.id}",
|
||||
)
|
||||
ui.button("Cancel", on_click=lambda: ui.navigate.to(f"/jobs/{job.id}"), icon="arrow_back").props("flat")
|
||||
|
||||
|
||||
# --- Helper Sub-Components ---
|
||||
@@ -488,17 +490,19 @@ def _render_upload_section(uploaded_files: list[tuple[str, bytes]]) -> None:
|
||||
"text-xs ui-text-danger"
|
||||
)
|
||||
|
||||
async def on_upload(event) -> None:
|
||||
async def on_upload(event: events.UploadEventArguments) -> None:
|
||||
payload = await event.file.read()
|
||||
uploaded_files.append((event.file.name, payload))
|
||||
ui.notify(f"Added {event.file.name}", type="positive")
|
||||
render_upload_list.refresh()
|
||||
|
||||
ui.upload(
|
||||
render_upload_picker(
|
||||
on_upload=on_upload,
|
||||
auto_upload=True,
|
||||
label="Select source files or a folder",
|
||||
).props('accept=".jpg,.jpeg,.png,.tif,.tiff,.pdf" webkitdirectory directory multiple').classes("w-full")
|
||||
extensions=SOURCE_UPLOAD_EXTENSIONS,
|
||||
directory=True,
|
||||
multiple=True,
|
||||
)
|
||||
|
||||
render_upload_list()
|
||||
|
||||
@@ -555,21 +559,5 @@ def _render_job_document_links(job: Job) -> None:
|
||||
).props("flat text-xs").classes("ui-link-primary w-full")
|
||||
|
||||
|
||||
def _parse_uuid(value: str | None) -> UUID | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return UUID(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_runtime_settings(request: Request) -> Settings:
|
||||
app_settings = getattr(request.app.state, "settings", None)
|
||||
if isinstance(app_settings, Settings):
|
||||
return app_settings
|
||||
return get_settings()
|
||||
|
||||
|
||||
def _latest_prompt_name(job: Job) -> str | None:
|
||||
return job.prompt_name
|
||||
|
||||
@@ -2,18 +2,15 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
from dataclasses import dataclass
|
||||
from uuid import UUID
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import Request
|
||||
from nicegui import events
|
||||
from nicegui import ui
|
||||
|
||||
from transcription.config import Settings
|
||||
from transcription.config import get_settings
|
||||
from transcription.db.models import Person
|
||||
from transcription.errors import ErrorCategory
|
||||
from transcription.services.people import PeopleError
|
||||
@@ -22,21 +19,47 @@ from transcription.services.people import PersonMediaError
|
||||
from transcription.services.people import store_person_portrait
|
||||
from transcription.ui.components.app_shell import render_navigation_header
|
||||
from transcription.ui.components.cards import archival_card
|
||||
from transcription.ui.components.confirm_delete import render_delete_actions
|
||||
from transcription.ui.components.data_display import metadata_row
|
||||
from transcription.ui.components.error_presenter import show_error
|
||||
from transcription.ui.components.formatters import compact_date
|
||||
from transcription.ui.components.formatters import family_search_url
|
||||
from transcription.ui.components.formatters import parse_iso_date
|
||||
from transcription.ui.components.guards import parsed_record_id
|
||||
from transcription.ui.components.guards import render_record_not_found
|
||||
from transcription.ui.components.media_urls import resolve_media_url
|
||||
from transcription.ui.components.primitives import destructive_button
|
||||
from transcription.ui.components.primitives import render_empty_state
|
||||
from transcription.ui.components.primitives import section_header_row
|
||||
from transcription.ui.components.table.people import PersonTableRow
|
||||
from transcription.ui.components.table.people import render_people_table
|
||||
from transcription.ui.components.upload_panel import IMAGE_UPLOAD_EXTENSIONS
|
||||
from transcription.ui.components.upload_panel import render_upload_picker
|
||||
from transcription.ui.components.viewers import dark_room_viewer
|
||||
from transcription.ui.runtime import resolve_runtime_settings
|
||||
from transcription.ui.theme import page_header
|
||||
|
||||
from ...db.session import SessionFactoryDep
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PersonFormFields:
|
||||
"""Bound input widgets for the Person create and edit forms."""
|
||||
|
||||
full_name: ui.input
|
||||
display_name: ui.input
|
||||
maiden_name: ui.input
|
||||
birth_date: ui.input
|
||||
birth_date_raw: ui.input
|
||||
birth_place: ui.input
|
||||
death_date: ui.input
|
||||
death_date_raw: ui.input
|
||||
death_place: ui.input
|
||||
biography: ui.textarea
|
||||
portrait_path: ui.input
|
||||
family_search_id: ui.input
|
||||
|
||||
|
||||
def register_page() -> None: # noqa: PLR0915
|
||||
"""Register people list and CRUD routes."""
|
||||
|
||||
@@ -92,28 +115,28 @@ def register_page() -> None: # noqa: PLR0915
|
||||
)
|
||||
|
||||
async def submit_create() -> None:
|
||||
full_name = (form["full_name"].value or "").strip()
|
||||
full_name = (form.full_name.value or "").strip()
|
||||
if not full_name:
|
||||
ui.notify("Full name is required.", type="warning")
|
||||
return
|
||||
|
||||
birth_date = _parse_iso_date(form["birth_date"].value)
|
||||
death_date = _parse_iso_date(form["death_date"].value)
|
||||
birth_date = parse_iso_date(form.birth_date.value)
|
||||
death_date = parse_iso_date(form.death_date.value)
|
||||
|
||||
candidate = Person(
|
||||
id=draft_person_id,
|
||||
full_name=full_name,
|
||||
display_name=(form["display_name"].value or "").strip() or None,
|
||||
maiden_name=(form["maiden_name"].value or "").strip() or None,
|
||||
display_name=(form.display_name.value or "").strip() or None,
|
||||
maiden_name=(form.maiden_name.value or "").strip() or None,
|
||||
birth_date=birth_date,
|
||||
birth_date_raw=(form["birth_date_raw"].value or "").strip() or None,
|
||||
birth_place=(form["birth_place"].value or "").strip() or None,
|
||||
birth_date_raw=(form.birth_date_raw.value or "").strip() or None,
|
||||
birth_place=(form.birth_place.value or "").strip() or None,
|
||||
death_date=death_date,
|
||||
death_date_raw=(form["death_date_raw"].value or "").strip() or None,
|
||||
death_place=(form["death_place"].value or "").strip() or None,
|
||||
biography=(form["biography"].value or "").strip() or None,
|
||||
portrait_path=(form["portrait_path"].value or "").strip() or None,
|
||||
family_search_id=(form["family_search_id"].value or "").strip() or None,
|
||||
death_date_raw=(form.death_date_raw.value or "").strip() or None,
|
||||
death_place=(form.death_place.value or "").strip() or None,
|
||||
biography=(form.biography.value or "").strip() or None,
|
||||
portrait_path=(form.portrait_path.value or "").strip() or None,
|
||||
family_search_id=(form.family_search_id.value or "").strip() or None,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -134,15 +157,14 @@ def register_page() -> None: # noqa: PLR0915
|
||||
people_service = PeopleService(session_factory=session_factory)
|
||||
render_navigation_header(current_path="/people")
|
||||
|
||||
parsed_person_id = _parse_uuid(person_id)
|
||||
parsed_person_id = parsed_record_id(person_id, noun="Person")
|
||||
if parsed_person_id is None:
|
||||
ui.label("Invalid person id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
person = await people_service.read_person_detail(parsed_person_id)
|
||||
except PeopleError:
|
||||
ui.label("Person not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Person")
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
show_error(exc, title="Load failed", operation="people.read")
|
||||
@@ -173,7 +195,7 @@ def register_page() -> None: # noqa: PLR0915
|
||||
with ui.grid().classes("w-full grid-cols-12 gap-4"):
|
||||
_render_person_portrait_zone(
|
||||
person,
|
||||
settings=_resolve_runtime_settings(request),
|
||||
settings=resolve_runtime_settings(request),
|
||||
request=request,
|
||||
)
|
||||
_render_person_biographical_zone(person)
|
||||
@@ -184,15 +206,14 @@ def register_page() -> None: # noqa: PLR0915
|
||||
people_service = PeopleService(session_factory=session_factory)
|
||||
render_navigation_header(current_path="/people")
|
||||
|
||||
parsed_person_id = _parse_uuid(person_id)
|
||||
parsed_person_id = parsed_record_id(person_id, noun="Person")
|
||||
if parsed_person_id is None:
|
||||
ui.label("Invalid person id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
person = await people_service.read_person_detail(parsed_person_id)
|
||||
except PeopleError:
|
||||
ui.label("Person not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Person")
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
show_error(exc, title="Load failed", operation="people.edit.read")
|
||||
@@ -208,28 +229,28 @@ def register_page() -> None: # noqa: PLR0915
|
||||
)
|
||||
|
||||
async def submit_edit() -> None:
|
||||
full_name = (form["full_name"].value or "").strip()
|
||||
full_name = (form.full_name.value or "").strip()
|
||||
if not full_name:
|
||||
ui.notify("Full name is required.", type="warning")
|
||||
return
|
||||
|
||||
birth_date = _parse_iso_date(form["birth_date"].value)
|
||||
death_date = _parse_iso_date(form["death_date"].value)
|
||||
birth_date = parse_iso_date(form.birth_date.value)
|
||||
death_date = parse_iso_date(form.death_date.value)
|
||||
|
||||
candidate = Person(
|
||||
id=person.id,
|
||||
full_name=full_name,
|
||||
display_name=(form["display_name"].value or "").strip() or None,
|
||||
maiden_name=(form["maiden_name"].value or "").strip() or None,
|
||||
display_name=(form.display_name.value or "").strip() or None,
|
||||
maiden_name=(form.maiden_name.value or "").strip() or None,
|
||||
birth_date=birth_date,
|
||||
birth_date_raw=(form["birth_date_raw"].value or "").strip() or None,
|
||||
birth_place=(form["birth_place"].value or "").strip() or None,
|
||||
birth_date_raw=(form.birth_date_raw.value or "").strip() or None,
|
||||
birth_place=(form.birth_place.value or "").strip() or None,
|
||||
death_date=death_date,
|
||||
death_date_raw=(form["death_date_raw"].value or "").strip() or None,
|
||||
death_place=(form["death_place"].value or "").strip() or None,
|
||||
biography=(form["biography"].value or "").strip() or None,
|
||||
portrait_path=(form["portrait_path"].value or "").strip() or None,
|
||||
family_search_id=(form["family_search_id"].value or "").strip() or None,
|
||||
death_date_raw=(form.death_date_raw.value or "").strip() or None,
|
||||
death_place=(form.death_place.value or "").strip() or None,
|
||||
biography=(form.biography.value or "").strip() or None,
|
||||
portrait_path=(form.portrait_path.value or "").strip() or None,
|
||||
family_search_id=(form.family_search_id.value or "").strip() or None,
|
||||
metadata_=person.metadata_,
|
||||
created_at=person.created_at,
|
||||
updated_at=person.updated_at,
|
||||
@@ -255,15 +276,14 @@ def register_page() -> None: # noqa: PLR0915
|
||||
people_service = PeopleService(session_factory=session_factory)
|
||||
render_navigation_header(current_path="/people")
|
||||
|
||||
parsed_person_id = _parse_uuid(person_id)
|
||||
parsed_person_id = parsed_record_id(person_id, noun="Person")
|
||||
if parsed_person_id is None:
|
||||
ui.label("Invalid person id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
person = await people_service.read_person_detail(parsed_person_id)
|
||||
except PeopleError:
|
||||
ui.label("Person not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Person")
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
show_error(exc, title="Load failed", operation="people.delete.read")
|
||||
@@ -301,12 +321,10 @@ def register_page() -> None: # noqa: PLR0915
|
||||
ui.notify("Person deleted", type="positive")
|
||||
ui.navigate.to("/people")
|
||||
|
||||
with ui.row().classes("w-full items-center gap-2 mt-2"):
|
||||
destructive_button(
|
||||
"Delete person permanently", on_click=submit_delete, icon="delete_forever", variant="solid"
|
||||
)
|
||||
ui.button("Cancel", on_click=lambda: ui.navigate.to(f"/people/{person.id}"), icon="arrow_back").props(
|
||||
"flat"
|
||||
render_delete_actions(
|
||||
confirm_label="Delete person permanently",
|
||||
on_confirm=submit_delete,
|
||||
cancel_target=f"/people/{person.id}",
|
||||
)
|
||||
|
||||
|
||||
@@ -318,7 +336,7 @@ def _render_person_form_fields(
|
||||
request: Request,
|
||||
person: Person | None = None,
|
||||
person_id: UUID,
|
||||
) -> dict[str, Any]:
|
||||
) -> PersonFormFields:
|
||||
with archival_card(extra_classes="gap-3"):
|
||||
with ui.row().classes("w-full gap-3 grid grid-cols-1 md:grid-cols-3"):
|
||||
full_name_input = (
|
||||
@@ -405,28 +423,32 @@ def _render_person_form_fields(
|
||||
|
||||
_bind_portrait_file_picker(
|
||||
portrait_path_input,
|
||||
settings=_resolve_runtime_settings(request),
|
||||
settings=resolve_runtime_settings(request),
|
||||
person_id=person_id,
|
||||
)
|
||||
|
||||
return {
|
||||
"full_name": full_name_input,
|
||||
"display_name": display_name_input,
|
||||
"maiden_name": maiden_name_input,
|
||||
"birth_date": birth_date_input,
|
||||
"birth_date_raw": birth_date_raw_input,
|
||||
"birth_place": birth_place_input,
|
||||
"death_date": death_date_input,
|
||||
"death_date_raw": death_date_raw_input,
|
||||
"death_place": death_place_input,
|
||||
"biography": biography_input,
|
||||
"portrait_path": portrait_path_input,
|
||||
"family_search_id": family_search_id_input,
|
||||
}
|
||||
return PersonFormFields(
|
||||
full_name=full_name_input,
|
||||
display_name=display_name_input,
|
||||
maiden_name=maiden_name_input,
|
||||
birth_date=birth_date_input,
|
||||
birth_date_raw=birth_date_raw_input,
|
||||
birth_place=birth_place_input,
|
||||
death_date=death_date_input,
|
||||
death_date_raw=death_date_raw_input,
|
||||
death_place=death_place_input,
|
||||
biography=biography_input,
|
||||
portrait_path=portrait_path_input,
|
||||
family_search_id=family_search_id_input,
|
||||
)
|
||||
|
||||
|
||||
def _render_person_portrait_zone(person: Person, *, settings: Settings, request: Request) -> None:
|
||||
portrait_src = _resolve_portrait_src(person.portrait_path, settings=settings, request=request)
|
||||
portrait_src = resolve_media_url(
|
||||
person.portrait_path,
|
||||
upload_dir=settings.upload_dir,
|
||||
base_url=str(request.base_url),
|
||||
)
|
||||
with ui.column().classes("col-span-12 lg:col-span-4"):
|
||||
dark_room_viewer(portrait_src, count_label="Portrait Media")
|
||||
|
||||
@@ -489,10 +511,10 @@ def _render_linked_documents(person: Person) -> None:
|
||||
|
||||
|
||||
def _bind_portrait_file_picker(portrait_path_input: ui.input, *, settings: Settings, person_id: UUID) -> None:
|
||||
async def on_portrait_selected(event) -> None:
|
||||
async def on_portrait_selected(event: events.UploadEventArguments) -> None:
|
||||
payload = await event.file.read()
|
||||
try:
|
||||
stored_path = store_person_portrait(
|
||||
stored_path = await store_person_portrait(
|
||||
person_id=person_id,
|
||||
filename=event.file.name,
|
||||
file_bytes=payload,
|
||||
@@ -501,8 +523,8 @@ def _bind_portrait_file_picker(portrait_path_input: ui.input, *, settings: Setti
|
||||
except PersonMediaError as exc:
|
||||
ui.notify(str(exc), type="negative")
|
||||
return
|
||||
except Exception: # noqa: BLE001
|
||||
ui.notify("Unable to store portrait image.", type="negative")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
show_error(exc, title="Upload failed", operation="people.portrait.store")
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -513,93 +535,10 @@ def _bind_portrait_file_picker(portrait_path_input: ui.input, *, settings: Setti
|
||||
portrait_path_input.value = relative_path
|
||||
ui.notify("Portrait uploaded.", type="positive")
|
||||
|
||||
ui.upload(
|
||||
render_upload_picker(
|
||||
on_upload=on_portrait_selected,
|
||||
auto_upload=True,
|
||||
label="Choose portrait file",
|
||||
).props('accept=".jpg,.jpeg,.png,.gif,.webp,.bmp,.tif,.tiff"').classes("w-full")
|
||||
extensions=IMAGE_UPLOAD_EXTENSIONS,
|
||||
)
|
||||
portrait_dir = settings.upload_dir / "persons" / str(person_id)
|
||||
ui.label(f"Portraits are stored under {portrait_dir}.").classes("text-xs ui-text-muted")
|
||||
|
||||
|
||||
def _resolve_portrait_src(path: str | None, *, settings: Settings, request: Request) -> str | None:
|
||||
candidate = (path or "").strip()
|
||||
if not candidate:
|
||||
return None
|
||||
|
||||
normalized = candidate.replace("\\", "/")
|
||||
lowered = normalized.casefold()
|
||||
if lowered.startswith(("http://", "https://", "data:")):
|
||||
return normalized
|
||||
if normalized.startswith("/uploads/"):
|
||||
return _to_absolute_upload_url(normalized, request=request)
|
||||
|
||||
upload_dir = settings.upload_dir.resolve()
|
||||
path_obj = Path(candidate)
|
||||
|
||||
if path_obj.is_absolute():
|
||||
absolute_candidates = [path_obj.resolve()]
|
||||
else:
|
||||
absolute_candidates = [
|
||||
(Path.cwd() / path_obj).resolve(),
|
||||
(upload_dir / path_obj).resolve(),
|
||||
]
|
||||
|
||||
for absolute_candidate in absolute_candidates:
|
||||
try:
|
||||
relative = absolute_candidate.relative_to(upload_dir).as_posix()
|
||||
return _to_absolute_upload_url(f"/uploads/{quote(relative)}", request=request)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
upload_name = upload_dir.name.casefold()
|
||||
normalized_parts = Path(normalized).parts
|
||||
lowered_parts = [part.casefold() for part in normalized_parts]
|
||||
if upload_name in lowered_parts:
|
||||
idx = lowered_parts.index(upload_name)
|
||||
relative = Path(*normalized_parts[idx + 1 :]).as_posix()
|
||||
if relative:
|
||||
return _to_absolute_upload_url(f"/uploads/{quote(relative)}", request=request)
|
||||
|
||||
if lowered.startswith("uploads/"):
|
||||
return _to_absolute_upload_url(f"/{normalized}", request=request)
|
||||
if lowered.startswith("data/"):
|
||||
relative = normalized.split("/", 1)[1] if "/" in normalized else ""
|
||||
if relative:
|
||||
return _to_absolute_upload_url(f"/uploads/{quote(relative)}", request=request)
|
||||
if lowered.startswith(("documents/", "persons/")):
|
||||
return _to_absolute_upload_url(f"/uploads/{quote(normalized)}", request=request)
|
||||
|
||||
return _to_absolute_upload_url(f"/uploads/{quote(path_obj.name)}", request=request)
|
||||
|
||||
|
||||
def _to_absolute_upload_url(path: str, *, request: Request) -> str:
|
||||
base = str(request.base_url).rstrip("/")
|
||||
normalized_path = path if path.startswith("/") else f"/{path}"
|
||||
return f"{base}{normalized_path}"
|
||||
|
||||
|
||||
def _resolve_runtime_settings(request: Request) -> Settings:
|
||||
app_settings = getattr(request.app.state, "settings", None)
|
||||
if isinstance(app_settings, Settings):
|
||||
return app_settings
|
||||
return get_settings()
|
||||
|
||||
|
||||
def _parse_uuid(value: str | None) -> UUID | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return UUID(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_iso_date(value: str | None) -> date | None:
|
||||
candidate = (value or "").strip()
|
||||
if not candidate:
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(candidate)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from nicegui import ui
|
||||
@@ -14,6 +15,8 @@ from transcription.services.documents import DocumentPrintSource
|
||||
from transcription.services.documents import DocumentService
|
||||
from transcription.ui.components.error_presenter import show_error
|
||||
from transcription.ui.components.formatters import compact_date
|
||||
from transcription.ui.components.guards import parsed_record_id
|
||||
from transcription.ui.components.guards import render_record_not_found
|
||||
from transcription.ui.theme import page_header
|
||||
|
||||
from ...db.session import SessionFactoryDep
|
||||
@@ -26,17 +29,15 @@ def register_page() -> None:
|
||||
|
||||
@ui.page("/documents/{document_id}/print")
|
||||
async def document_print_preview_page(document_id: str, session_factory: SessionFactoryDep) -> None:
|
||||
try:
|
||||
parsed_id = UUID(document_id)
|
||||
except ValueError:
|
||||
ui.label("Invalid document id").classes("text-h6 ui-text-danger p-4")
|
||||
parsed_id = parsed_record_id(document_id, noun="Document")
|
||||
if parsed_id is None:
|
||||
return
|
||||
|
||||
service = DocumentService(session_factory=session_factory)
|
||||
try:
|
||||
projection = await service.read_document_print_projection(parsed_id)
|
||||
except DocumentError:
|
||||
ui.label("Document not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Document")
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
show_error(exc, title="Print preview unavailable", operation="documents.print.read")
|
||||
@@ -111,6 +112,30 @@ def _render_facsimile_source(*, document_id: UUID, source: DocumentPrintSource,
|
||||
ui.label(text).classes("print-transcription print-preserve-lines")
|
||||
|
||||
|
||||
def _render_print_table(
|
||||
*,
|
||||
columns: list[dict[str, Any]],
|
||||
rows: list[dict[str, Any]],
|
||||
extra_classes: str,
|
||||
hide_header: bool,
|
||||
) -> None:
|
||||
"""Render one unpaginated print-layout table.
|
||||
|
||||
Print tables deliberately bypass `build_table`: they carry print-only styling,
|
||||
must never paginate or expose a search box, and are rendered for a static
|
||||
document rather than an interactive page.
|
||||
"""
|
||||
props = ["flat", "hide-bottom"]
|
||||
if hide_header:
|
||||
props.insert(1, "hide-header")
|
||||
ui.table(
|
||||
columns=columns,
|
||||
rows=rows,
|
||||
row_key="field",
|
||||
pagination={"rowsPerPage": 0},
|
||||
).props(" ".join(props)).classes(f"print-data-table {extra_classes}")
|
||||
|
||||
|
||||
def _render_metadata_table(projection: DocumentPrintProjection) -> None:
|
||||
rows = [
|
||||
{"field": "Author", "value": ", ".join(projection.authors) or "Not set"},
|
||||
@@ -122,19 +147,19 @@ def _render_metadata_table(projection: DocumentPrintProjection) -> None:
|
||||
{"field": "Location Created", "value": projection.location_created or "Not set"},
|
||||
{"field": "Archival Identifier", "value": projection.archive_identifier or "Not set"},
|
||||
]
|
||||
ui.table(
|
||||
_render_print_table(
|
||||
columns=[
|
||||
{"name": "field", "label": "", "field": "field", "align": "left"},
|
||||
{"name": "value", "label": "", "field": "value", "align": "left"},
|
||||
],
|
||||
rows=rows,
|
||||
row_key="field",
|
||||
pagination={"rowsPerPage": 0},
|
||||
).props("flat hide-header hide-bottom").classes("print-data-table print-metadata-table")
|
||||
extra_classes="print-metadata-table",
|
||||
hide_header=True,
|
||||
)
|
||||
|
||||
|
||||
def _render_job_table(jobs: tuple[DocumentPrintJob, ...]) -> None:
|
||||
columns = [{"name": "field", "label": "", "field": "field", "align": "left"}]
|
||||
columns: list[dict[str, Any]] = [{"name": "field", "label": "", "field": "field", "align": "left"}]
|
||||
for index in range(1, len(jobs) + 1):
|
||||
columns.append({"name": f"job_{index}", "label": f"Job {index}", "field": f"job_{index}", "align": "left"})
|
||||
fields = (
|
||||
@@ -153,12 +178,12 @@ def _render_job_table(jobs: tuple[DocumentPrintJob, ...]) -> None:
|
||||
}
|
||||
for field, value in fields
|
||||
]
|
||||
ui.table(
|
||||
_render_print_table(
|
||||
columns=columns,
|
||||
rows=rows,
|
||||
row_key="field",
|
||||
pagination={"rowsPerPage": 0},
|
||||
).props("flat hide-bottom").classes("print-data-table print-job-table")
|
||||
extra_classes="print-job-table",
|
||||
hide_header=False,
|
||||
)
|
||||
|
||||
|
||||
def reflow_transcription(text: str) -> list[str]:
|
||||
|
||||
@@ -17,6 +17,7 @@ from transcription.ui.components.error_presenter import show_error
|
||||
from transcription.ui.components.primitives import destructive_button
|
||||
from transcription.ui.components.primitives import render_empty_state
|
||||
from transcription.ui.components.primitives import section_header_row
|
||||
from transcription.ui.components.table.registry import render_registry_table
|
||||
from transcription.ui.theme import page_header
|
||||
|
||||
from ...db.session import SessionFactoryDep
|
||||
@@ -62,55 +63,10 @@ def register_page(*, settings: Settings) -> None: # noqa: PLR0915
|
||||
}
|
||||
for item in document_types
|
||||
]
|
||||
table = ui.table(
|
||||
columns=[
|
||||
{
|
||||
"name": "label",
|
||||
"label": "Label",
|
||||
"field": "label",
|
||||
"align": "left",
|
||||
"sortable": True,
|
||||
},
|
||||
{
|
||||
"name": "document_count",
|
||||
"label": "Documents",
|
||||
"field": "document_count",
|
||||
"align": "right",
|
||||
"sortable": True,
|
||||
},
|
||||
{
|
||||
"name": "is_active",
|
||||
"label": "Active",
|
||||
"field": "is_active",
|
||||
"align": "center",
|
||||
},
|
||||
{
|
||||
"name": "is_built_in",
|
||||
"label": "Built-in",
|
||||
"field": "is_built_in",
|
||||
"align": "center",
|
||||
},
|
||||
],
|
||||
rows=rows,
|
||||
row_key="id",
|
||||
selection="single",
|
||||
pagination={"rowsPerPage": 0, "sortBy": "label"},
|
||||
).classes("w-full ui-table")
|
||||
table.add_slot(
|
||||
"body-cell-is_active",
|
||||
"""
|
||||
<q-td :props="props" class="text-center">
|
||||
<q-checkbox :model-value="props.value" disable dense />
|
||||
</q-td>
|
||||
""",
|
||||
)
|
||||
table.add_slot(
|
||||
"body-cell-is_built_in",
|
||||
"""
|
||||
<q-td :props="props" class="text-center">
|
||||
{{ props.value ? 'Yes' : 'No' }}
|
||||
</q-td>
|
||||
""",
|
||||
table = render_registry_table(
|
||||
rows,
|
||||
count_field="document_count",
|
||||
count_label="Documents",
|
||||
)
|
||||
|
||||
async def save_type(
|
||||
@@ -214,44 +170,10 @@ def register_page(*, settings: Settings) -> None: # noqa: PLR0915
|
||||
}
|
||||
for role in roles
|
||||
]
|
||||
table = ui.table(
|
||||
columns=[
|
||||
{"name": "label", "label": "Label", "field": "label", "align": "left", "sortable": True},
|
||||
{
|
||||
"name": "link_count",
|
||||
"label": "Links",
|
||||
"field": "link_count",
|
||||
"align": "right",
|
||||
"sortable": True,
|
||||
},
|
||||
{"name": "is_active", "label": "Active", "field": "is_active", "align": "center"},
|
||||
{
|
||||
"name": "is_built_in",
|
||||
"label": "Built-in",
|
||||
"field": "is_built_in",
|
||||
"align": "center",
|
||||
},
|
||||
],
|
||||
rows=rows,
|
||||
row_key="id",
|
||||
selection="single",
|
||||
pagination={"rowsPerPage": 0, "sortBy": "label"},
|
||||
).classes("w-full ui-table")
|
||||
table.add_slot(
|
||||
"body-cell-is_active",
|
||||
"""
|
||||
<q-td :props="props" class="text-center">
|
||||
<q-checkbox :model-value="props.value" disable dense />
|
||||
</q-td>
|
||||
""",
|
||||
)
|
||||
table.add_slot(
|
||||
"body-cell-is_built_in",
|
||||
"""
|
||||
<q-td :props="props" class="text-center">
|
||||
{{ props.value ? 'Yes' : 'No' }}
|
||||
</q-td>
|
||||
""",
|
||||
table = render_registry_table(
|
||||
rows,
|
||||
count_field="link_count",
|
||||
count_label="Links",
|
||||
)
|
||||
|
||||
async def save_role(
|
||||
|
||||
@@ -4,34 +4,38 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import Request
|
||||
from nicegui import ui
|
||||
from sqlalchemy import inspect as sqlalchemy_inspect
|
||||
|
||||
from transcription.config import Settings
|
||||
from transcription.config import get_settings
|
||||
from transcription.db.models import ExecutionAttempt
|
||||
from transcription.db.models import JobSource
|
||||
from transcription.db.models import ProcessingArtifact
|
||||
from transcription.db.models import Source
|
||||
from transcription.services.sources import LatestExecutionAttempt
|
||||
from transcription.services.sources import SourceDeleteBlockedError
|
||||
from transcription.services.sources import SourceService
|
||||
from transcription.services.sources import TranscriptionNotFoundError
|
||||
from transcription.ui.components.app_shell import render_navigation_header
|
||||
from transcription.ui.components.cards import archival_card
|
||||
from transcription.ui.components.confirm_delete import render_delete_actions
|
||||
from transcription.ui.components.confirm_delete import render_delete_blocked_notice
|
||||
from transcription.ui.components.data_display import archival_badge
|
||||
from transcription.ui.components.data_display import metadata_row
|
||||
from transcription.ui.components.error_presenter import show_error
|
||||
from transcription.ui.components.formatters import parse_uuid
|
||||
from transcription.ui.components.guards import parsed_record_id
|
||||
from transcription.ui.components.guards import render_record_not_found
|
||||
from transcription.ui.components.media_urls import resolve_media_url
|
||||
from transcription.ui.components.primitives import destructive_button
|
||||
from transcription.ui.components.primitives import render_empty_state
|
||||
from transcription.ui.components.primitives import section_header_row
|
||||
from transcription.ui.components.table.sources import SourceTableRow
|
||||
from transcription.ui.components.table.sources import render_sources_table
|
||||
from transcription.ui.components.viewers import dark_room_viewer
|
||||
from transcription.ui.runtime import resolve_runtime_settings
|
||||
from transcription.ui.theme import page_header
|
||||
|
||||
from ...db.session import SessionFactoryDep
|
||||
@@ -47,8 +51,8 @@ def register_page() -> None: # noqa: PLR0915
|
||||
job_id: str | None = None,
|
||||
) -> None:
|
||||
sources_service = SourceService(session_factory=session_factory)
|
||||
parsed_doc_id = _parse_uuid(document_id)
|
||||
parsed_job_id = _parse_uuid(job_id)
|
||||
parsed_doc_id = parse_uuid(document_id)
|
||||
parsed_job_id = parse_uuid(job_id)
|
||||
|
||||
header_title = "Source Asset Records"
|
||||
if parsed_doc_id is not None:
|
||||
@@ -108,12 +112,10 @@ def register_page() -> None: # noqa: PLR0915
|
||||
@ui.page("/sources/{source_id}")
|
||||
async def source_detail_page(source_id: str, request: Request, session_factory: SessionFactoryDep) -> None:
|
||||
sources_service = SourceService(session_factory=session_factory)
|
||||
parsed_source_id = _parse_uuid(source_id)
|
||||
|
||||
render_navigation_header(current_path="/sources")
|
||||
|
||||
parsed_source_id = parsed_record_id(source_id, noun="Source")
|
||||
if parsed_source_id is None:
|
||||
ui.label("Invalid source id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -130,7 +132,7 @@ def register_page() -> None: # noqa: PLR0915
|
||||
)
|
||||
attempts = list(await sources_service.list_execution_attempts(source_id=parsed_source_id))
|
||||
except TranscriptionNotFoundError:
|
||||
ui.label("Source not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Source")
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
show_error(exc, title="Load failed", operation="sources.read")
|
||||
@@ -180,7 +182,7 @@ def register_page() -> None: # noqa: PLR0915
|
||||
_render_source_navigation(navigation.previous_id, navigation.next_id)
|
||||
_render_source_viewer_zone(
|
||||
source,
|
||||
settings=_resolve_runtime_settings(request),
|
||||
settings=resolve_runtime_settings(request),
|
||||
request=request,
|
||||
)
|
||||
_render_source_transcription_column(
|
||||
@@ -204,18 +206,16 @@ def register_page() -> None: # noqa: PLR0915
|
||||
@ui.page("/sources/{source_id}/delete")
|
||||
async def source_delete_page(source_id: str, session_factory: SessionFactoryDep) -> None:
|
||||
sources_service = SourceService(session_factory=session_factory)
|
||||
parsed_source_id = _parse_uuid(source_id)
|
||||
|
||||
render_navigation_header(current_path="/sources")
|
||||
|
||||
parsed_source_id = parsed_record_id(source_id, noun="Source")
|
||||
if parsed_source_id is None:
|
||||
ui.label("Invalid source id").classes("text-h6 ui-text-danger p-4")
|
||||
return
|
||||
|
||||
try:
|
||||
source = await sources_service.read_source_detail(parsed_source_id)
|
||||
except TranscriptionNotFoundError:
|
||||
ui.label("Source not found").classes("text-h6 ui-text-danger p-4")
|
||||
render_record_not_found("Source")
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
show_error(exc, title="Load failed", operation="sources.delete.read")
|
||||
@@ -228,23 +228,12 @@ def register_page() -> None: # noqa: PLR0915
|
||||
ui.label(f"Source: {source.upload_name}").classes("text-sm font-semibold ui-text-primary")
|
||||
|
||||
if source.job_sources:
|
||||
ui.label("Delete is only available for unlinked sources.").classes(
|
||||
"text-xs ui-text-danger font-bold mt-2"
|
||||
render_delete_blocked_notice(
|
||||
reason="Delete is only available for unlinked sources.",
|
||||
guidance="Open the related job record and remove job links first.",
|
||||
back_label="Back to Source",
|
||||
back_target=f"/sources/{source.id}",
|
||||
)
|
||||
ui.label("Open the related job record and remove job links first.").classes(
|
||||
"text-xs ui-text-muted italic"
|
||||
)
|
||||
with ui.row().classes("w-full items-center gap-2 mt-4"):
|
||||
ui.button(
|
||||
"Back to Source",
|
||||
on_click=lambda: ui.navigate.to(f"/sources/{source.id}"),
|
||||
icon="arrow_back",
|
||||
).classes("ui-btn-primary text-xs")
|
||||
ui.button(
|
||||
"Go to Jobs",
|
||||
on_click=lambda: ui.navigate.to("/jobs"),
|
||||
icon="work_history",
|
||||
).props("flat text-xs")
|
||||
return
|
||||
|
||||
ui.label("This action permanently deletes the source record.").classes(
|
||||
@@ -268,18 +257,16 @@ def register_page() -> None: # noqa: PLR0915
|
||||
ui.notify("Source deleted", type="positive")
|
||||
ui.navigate.to("/sources")
|
||||
|
||||
with ui.row().classes("w-full items-center gap-2 mt-2"):
|
||||
destructive_button(
|
||||
"Delete source permanently", on_click=submit_delete, icon="delete_forever", variant="solid"
|
||||
)
|
||||
ui.button("Cancel", on_click=lambda: ui.navigate.to(f"/sources/{source.id}"), icon="arrow_back").props(
|
||||
"flat"
|
||||
render_delete_actions(
|
||||
confirm_label="Delete source permanently",
|
||||
on_confirm=submit_delete,
|
||||
cancel_target=f"/sources/{source.id}",
|
||||
)
|
||||
|
||||
|
||||
def _render_source_viewer_zone(source: Source, *, settings: Settings, request: Request) -> None:
|
||||
dark_room_viewer(
|
||||
_resolve_source_media_src(source.file_path, settings=settings, request=request),
|
||||
resolve_media_url(source.file_path, upload_dir=settings.upload_dir, base_url=str(request.base_url)),
|
||||
count_label=f"Page {source.page_number}",
|
||||
)
|
||||
|
||||
@@ -323,7 +310,7 @@ def _render_source_metadata_column(
|
||||
*,
|
||||
source: Source,
|
||||
latest_job_source: JobSource | None,
|
||||
latest_attempt: ExecutionAttempt | None,
|
||||
latest_attempt: LatestExecutionAttempt | None,
|
||||
source_artifacts: list[ProcessingArtifact],
|
||||
) -> None:
|
||||
with ui.column().classes("col-span-12 lg:col-span-4 gap-4"):
|
||||
@@ -349,7 +336,7 @@ def _render_source_metadata_zone(source: Source) -> None:
|
||||
def _render_source_job_metadata_zone(
|
||||
latest_job_source: JobSource | None,
|
||||
*,
|
||||
latest_attempt: ExecutionAttempt | None,
|
||||
latest_attempt: LatestExecutionAttempt | None,
|
||||
source_artifacts: list[ProcessingArtifact],
|
||||
) -> None:
|
||||
with archival_card(title="SourceJob Metadata"):
|
||||
@@ -394,7 +381,7 @@ def _render_source_job_metadata_zone(
|
||||
def _render_provider_evidence(
|
||||
job_source: JobSource,
|
||||
*,
|
||||
latest_attempt: ExecutionAttempt | None,
|
||||
latest_attempt: LatestExecutionAttempt | None,
|
||||
source_artifacts: list[ProcessingArtifact],
|
||||
) -> None:
|
||||
ui.label("Provider Evidence").classes("text-xs font-semibold ui-text-primary mt-3")
|
||||
@@ -408,11 +395,11 @@ def _render_provider_evidence(
|
||||
_render_json_evidence("Derived Artifacts", _artifact_display(source_artifacts))
|
||||
return
|
||||
|
||||
attempt = latest_attempt
|
||||
attempt = latest_attempt.attempt
|
||||
metadata_row("Attempt:", str(attempt.attempt_number))
|
||||
metadata_row("Duration:", f"{attempt.duration_ms} ms")
|
||||
_render_json_evidence("Request Manifest", attempt.request_manifest)
|
||||
_render_json_evidence("Transport Response", _transport_display(attempt))
|
||||
_render_json_evidence("Transport Response", _transport_display(latest_attempt))
|
||||
_render_json_evidence("OpenRouter SDK Response Snapshot", attempt.sdk_response_snapshot)
|
||||
_render_json_evidence("Normalized Metadata", attempt.normalized_metadata)
|
||||
_render_json_evidence("Software Context", attempt.software_context)
|
||||
@@ -420,7 +407,7 @@ def _render_provider_evidence(
|
||||
|
||||
|
||||
def _artifact_display(artifacts: list[ProcessingArtifact]) -> list[dict[str, object]] | None:
|
||||
payload = [
|
||||
payload: list[dict[str, object]] = [
|
||||
{
|
||||
"id": str(artifact.id),
|
||||
"type": artifact.artifact_type,
|
||||
@@ -434,10 +421,10 @@ def _artifact_display(artifacts: list[ProcessingArtifact]) -> list[dict[str, obj
|
||||
return payload or None
|
||||
|
||||
|
||||
def _transport_display(attempt: ExecutionAttempt) -> dict[str, object]:
|
||||
def _transport_display(latest_attempt: LatestExecutionAttempt) -> dict[str, object]:
|
||||
attempt = latest_attempt.attempt
|
||||
body: object | None = None
|
||||
body_is_deferred = "transport_body" in sqlalchemy_inspect(attempt).unloaded
|
||||
if body_is_deferred:
|
||||
if latest_attempt.transport_body_deferred:
|
||||
body = "Omitted from Source Detail; use Export Evidence to retrieve the exact bytes."
|
||||
elif attempt.transport_body is not None:
|
||||
try:
|
||||
@@ -706,81 +693,3 @@ def _resolve_original_transcription(*, source: Source, latest_job_source: JobSou
|
||||
if source.raw_transcription is None and latest_job_source is not None:
|
||||
return latest_job_source.raw_transcription
|
||||
return source.raw_transcription
|
||||
|
||||
|
||||
def _resolve_source_media_src(path: str | None, *, settings: Settings, request: Request) -> str | None:
|
||||
candidate = (path or "").strip()
|
||||
if not candidate:
|
||||
return None
|
||||
|
||||
normalized = candidate.replace("\\", "/")
|
||||
lowered = normalized.casefold()
|
||||
|
||||
if lowered.startswith(("http://", "https://", "data:")):
|
||||
return normalized
|
||||
if normalized.startswith("/uploads/"):
|
||||
return _to_absolute_upload_url(normalized, request=request)
|
||||
|
||||
upload_dir = settings.upload_dir.resolve()
|
||||
path_obj = Path(candidate)
|
||||
|
||||
# Case 1: absolute filesystem path
|
||||
if path_obj.is_absolute():
|
||||
absolute_candidates = [path_obj.resolve()]
|
||||
else:
|
||||
# Case 2: relative path that may already include data root name (e.g. data/documents/...)
|
||||
absolute_candidates = [
|
||||
(Path.cwd() / path_obj).resolve(),
|
||||
(upload_dir / path_obj).resolve(),
|
||||
]
|
||||
|
||||
for absolute_candidate in absolute_candidates:
|
||||
try:
|
||||
relative = absolute_candidate.relative_to(upload_dir).as_posix()
|
||||
return _to_absolute_upload_url(f"/uploads/{quote(relative)}", request=request)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
# Fallback: if path contains upload-dir folder name, strip through that segment.
|
||||
upload_name = upload_dir.name.casefold()
|
||||
normalized_parts = Path(normalized).parts
|
||||
lowered_parts = [part.casefold() for part in normalized_parts]
|
||||
if upload_name in lowered_parts:
|
||||
idx = lowered_parts.index(upload_name)
|
||||
relative = Path(*normalized_parts[idx + 1 :]).as_posix()
|
||||
if relative:
|
||||
return _to_absolute_upload_url(f"/uploads/{quote(relative)}", request=request)
|
||||
|
||||
# Final fallback: treat as already relative to upload root.
|
||||
if lowered.startswith("uploads/"):
|
||||
return _to_absolute_upload_url(f"/{normalized}", request=request)
|
||||
if lowered.startswith("data/"):
|
||||
relative = normalized.split("/", 1)[1] if "/" in normalized else ""
|
||||
if relative:
|
||||
return _to_absolute_upload_url(f"/uploads/{quote(relative)}", request=request)
|
||||
if lowered.startswith(("documents/", "persons/")):
|
||||
return _to_absolute_upload_url(f"/uploads/{quote(normalized)}", request=request)
|
||||
|
||||
return _to_absolute_upload_url(f"/uploads/{quote(path_obj.name)}", request=request)
|
||||
|
||||
|
||||
def _to_absolute_upload_url(path: str, *, request: Request) -> str:
|
||||
base = str(request.base_url).rstrip("/")
|
||||
normalized_path = path if path.startswith("/") else f"/{path}"
|
||||
return f"{base}{normalized_path}"
|
||||
|
||||
|
||||
def _resolve_runtime_settings(request: Request) -> Settings:
|
||||
app_settings = getattr(request.app.state, "settings", None)
|
||||
if isinstance(app_settings, Settings):
|
||||
return app_settings
|
||||
return get_settings()
|
||||
|
||||
|
||||
def _parse_uuid(value: str | None) -> UUID | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return UUID(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@@ -10,9 +10,19 @@ from pathlib import PurePosixPath
|
||||
@cache
|
||||
def read_css(relative_path: str) -> str:
|
||||
"""Read and cache a CSS resource relative to ``ui/static``."""
|
||||
return _read_static(relative_path, suffix=".css")
|
||||
|
||||
|
||||
@cache
|
||||
def read_svg(relative_path: str) -> str:
|
||||
"""Read and cache an SVG resource relative to ``ui/static``."""
|
||||
return _read_static(relative_path, suffix=".svg")
|
||||
|
||||
|
||||
def _read_static(relative_path: str, *, suffix: str) -> str:
|
||||
resource_path = PurePosixPath(relative_path)
|
||||
if resource_path.is_absolute() or ".." in resource_path.parts or resource_path.suffix != ".css":
|
||||
msg = f"Invalid CSS resource path: {relative_path}"
|
||||
if resource_path.is_absolute() or ".." in resource_path.parts or resource_path.suffix != suffix:
|
||||
msg = f"Invalid {suffix.lstrip('.').upper()} resource path: {relative_path}"
|
||||
raise ValueError(msg)
|
||||
|
||||
resource = files("transcription.ui").joinpath("static", *resource_path.parts)
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
"""Request-scoped runtime resolution shared by page modules."""
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from transcription.config import Settings
|
||||
from transcription.config import get_settings
|
||||
|
||||
|
||||
def resolve_runtime_settings(request: Request) -> Settings:
|
||||
"""Prefer settings installed on application state, falling back to the default."""
|
||||
app_settings = getattr(request.app.state, "settings", None)
|
||||
if isinstance(app_settings, Settings):
|
||||
return app_settings
|
||||
return get_settings()
|
||||
@@ -461,46 +461,6 @@ input:focus-visible,
|
||||
min-height: 31.25rem;
|
||||
}
|
||||
|
||||
.document-panzoom-filename {
|
||||
max-width: 60%;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.document-panzoom-host {
|
||||
height: min(70vh, 52rem);
|
||||
overflow: hidden;
|
||||
touch-action: none;
|
||||
border: 1px solid var(--theme-viewer-border);
|
||||
border-radius: 0.125rem;
|
||||
background: var(--theme-viewer);
|
||||
}
|
||||
|
||||
.document-panzoom-surface {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: flex-start;
|
||||
}
|
||||
|
||||
.document-panzoom-media {
|
||||
width: auto;
|
||||
height: auto;
|
||||
display: block;
|
||||
max-width: 100%;
|
||||
max-height: 100%;
|
||||
user-select: none;
|
||||
-webkit-user-drag: none;
|
||||
}
|
||||
|
||||
.document-panzoom-iframe {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
border: 0;
|
||||
pointer-events: none;
|
||||
background: var(--theme-surface-raised);
|
||||
}
|
||||
|
||||
@media (max-width: 700px) {
|
||||
.app-shell {
|
||||
padding-inline: 0.75rem;
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 23 KiB |
File diff suppressed because one or more lines are too long
+36
-20
@@ -9,6 +9,7 @@ from contextlib import asynccontextmanager
|
||||
from contextlib import contextmanager
|
||||
from contextlib import suppress
|
||||
from typing import Protocol
|
||||
from typing import runtime_checkable
|
||||
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
@@ -18,15 +19,12 @@ from transcription.errors import AppError
|
||||
from transcription.errors import classify_unexpected_error
|
||||
|
||||
from .services import ServiceBundle
|
||||
from .services.documents import DocumentService
|
||||
from .services.jobs import JobService
|
||||
from .services.people import PeopleService
|
||||
from .services.sources import SourceService
|
||||
from .services.workflows import process_next_queued_job as process_next_queued_job_workflow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class WorkerNotifier(Protocol):
|
||||
"""Abstraction for signaling the worker loop about new work."""
|
||||
|
||||
@@ -54,11 +52,11 @@ class NoopWorkerNotifier:
|
||||
def resolve_worker_notifier(state: object) -> WorkerNotifier:
|
||||
"""Resolve notifier from app-like state objects with no-op fallback."""
|
||||
notifier = getattr(state, "worker_notifier", None)
|
||||
if isinstance(notifier, NoopWorkerNotifier):
|
||||
if isinstance(notifier, WorkerNotifier):
|
||||
return notifier
|
||||
if notifier is None:
|
||||
if notifier is not None:
|
||||
logger.warning("Ignoring worker_notifier of unsupported type %r; using no-op fallback.", type(notifier))
|
||||
return NoopWorkerNotifier()
|
||||
return notifier
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
@@ -119,7 +117,13 @@ async def run_worker_loop(
|
||||
|
||||
If wake_event is provided, signal activity wakes the loop immediately while
|
||||
timeout-based wakeups preserve current polling behavior.
|
||||
|
||||
The service bundle — and with it the provider's pooled HTTP client — is built
|
||||
once for the lifetime of the loop, so consecutive jobs reuse one connection
|
||||
instead of paying a fresh TLS handshake each time.
|
||||
"""
|
||||
services = ServiceBundle.from_session_factory(session_factory)
|
||||
try:
|
||||
while True:
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
logger.info("Worker stop event received")
|
||||
@@ -133,7 +137,10 @@ async def run_worker_loop(
|
||||
processed_any = False
|
||||
while True:
|
||||
with handle_worker_exceptions(operation="worker.process_next_queued_job"):
|
||||
processed = await process_next_queued_job(session_factory=session_factory)
|
||||
processed = await process_next_queued_job(
|
||||
session_factory=session_factory,
|
||||
services=services,
|
||||
)
|
||||
if not processed:
|
||||
break
|
||||
processed_any = True
|
||||
@@ -143,32 +150,41 @@ async def run_worker_loop(
|
||||
|
||||
if wake_event is None and not processed_any:
|
||||
await asyncio.sleep(poll_interval_seconds)
|
||||
finally:
|
||||
await services.aclose()
|
||||
|
||||
|
||||
async def process_next_queued_job(
|
||||
*,
|
||||
session: AsyncSession | None = None,
|
||||
session_factory: async_sessionmaker[AsyncSession] | None = None,
|
||||
services: ServiceBundle | None = None,
|
||||
) -> bool:
|
||||
"""Process the next queued job and persist terminal outcome.
|
||||
|
||||
Returns True when a job was processed, False when no queued job exists.
|
||||
"""
|
||||
if session_factory is None:
|
||||
services = ServiceBundle()
|
||||
else:
|
||||
services = ServiceBundle(
|
||||
documents=DocumentService(session_factory=session_factory),
|
||||
sources=SourceService(session_factory=session_factory),
|
||||
jobs=JobService(session_factory=session_factory),
|
||||
people=PeopleService(session_factory=session_factory),
|
||||
)
|
||||
|
||||
When ``services`` is supplied the caller owns its lifecycle; otherwise a
|
||||
bundle is created and closed here.
|
||||
"""
|
||||
if services is not None:
|
||||
return await _process_next_queued_job(services=services, session=session, session_factory=session_factory)
|
||||
|
||||
owned = ServiceBundle.from_session_factory(session_factory)
|
||||
try:
|
||||
return await _process_next_queued_job(services=owned, session=session, session_factory=session_factory)
|
||||
finally:
|
||||
await owned.aclose()
|
||||
|
||||
|
||||
async def _process_next_queued_job(
|
||||
*,
|
||||
services: ServiceBundle,
|
||||
session: AsyncSession | None,
|
||||
session_factory: async_sessionmaker[AsyncSession] | None,
|
||||
) -> bool:
|
||||
if session is None:
|
||||
async with session_scope(session_factory=session_factory) as local_session:
|
||||
return await process_next_queued_job_workflow(services=services, session=local_session)
|
||||
|
||||
return await process_next_queued_job_workflow(services=services, session=session)
|
||||
finally:
|
||||
await services.sources.aclose()
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
from openrouter import OpenRouter
|
||||
|
||||
from transcription.config import Settings
|
||||
from transcription.providers.base import ProviderError
|
||||
@@ -13,7 +15,7 @@ from transcription.providers.openrouter import OpenRouterTranscriptionProvider
|
||||
|
||||
|
||||
class _FakeChat:
|
||||
def __init__(self, response=None, error: Exception | None = None):
|
||||
def __init__(self, response=None, error: BaseException | None = None):
|
||||
self._response = response
|
||||
self._error = error
|
||||
self.calls = []
|
||||
@@ -26,10 +28,15 @@ class _FakeChat:
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, response=None, error: Exception | None = None):
|
||||
def __init__(self, response=None, error: BaseException | None = None):
|
||||
self.chat = _FakeChat(response=response, error=error)
|
||||
|
||||
|
||||
def _fake_client(response=None, error: BaseException | None = None) -> OpenRouter:
|
||||
"""Return a stub typed as the SDK client the provider declares."""
|
||||
return cast("OpenRouter", _FakeClient(response=response, error=error))
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestOpenRouterProviderInit:
|
||||
"""Verify OpenRouter provider initialization behavior."""
|
||||
@@ -37,13 +44,13 @@ class TestOpenRouterProviderInit:
|
||||
def test_model_falls_back_to_default_when_unset(self):
|
||||
"""Provider uses adapter default model when provider_model is None."""
|
||||
settings = Settings(openrouter_api_key="test-key", provider_model=None)
|
||||
provider = OpenRouterTranscriptionProvider(settings=settings, client=_FakeClient())
|
||||
provider = OpenRouterTranscriptionProvider(settings=settings, client=_fake_client())
|
||||
assert provider.model == DEFAULT_OPENROUTER_MODEL
|
||||
|
||||
def test_model_uses_configured_value(self):
|
||||
"""Provider uses configured provider_model when present."""
|
||||
settings = Settings(openrouter_api_key="test-key", provider_model="vendor/custom-model")
|
||||
provider = OpenRouterTranscriptionProvider(settings=settings, client=_FakeClient())
|
||||
provider = OpenRouterTranscriptionProvider(settings=settings, client=_fake_client())
|
||||
assert provider.model == "vendor/custom-model"
|
||||
|
||||
|
||||
@@ -61,7 +68,7 @@ class TestOpenRouterProviderTranscribe:
|
||||
openrouter_http_referer="https://example.test",
|
||||
openrouter_app_title="Transcription App",
|
||||
)
|
||||
provider = OpenRouterTranscriptionProvider(settings=settings, client=client)
|
||||
provider = OpenRouterTranscriptionProvider(settings=settings, client=cast("OpenRouter", client))
|
||||
|
||||
result = await provider.transcribe(
|
||||
prompt_text="Prompt body",
|
||||
@@ -79,7 +86,10 @@ class TestOpenRouterProviderTranscribe:
|
||||
"""Transcribe passes configured sampling parameters through to OpenRouter."""
|
||||
response = {"model": "vendor/model-a", "choices": [{"message": {"content": "Transcript text"}}]}
|
||||
client = _FakeClient(response=response)
|
||||
provider = OpenRouterTranscriptionProvider(settings=Settings(openrouter_api_key="test-key"), client=client)
|
||||
provider = OpenRouterTranscriptionProvider(
|
||||
settings=Settings(openrouter_api_key="test-key"),
|
||||
client=cast("OpenRouter", client),
|
||||
)
|
||||
|
||||
result = await provider.transcribe(
|
||||
prompt_text="Prompt body",
|
||||
@@ -110,7 +120,7 @@ class TestOpenRouterProviderTranscribe:
|
||||
}
|
||||
provider = OpenRouterTranscriptionProvider(
|
||||
settings=Settings(openrouter_api_key="test-key"),
|
||||
client=_FakeClient(response=response),
|
||||
client=_fake_client(response=response),
|
||||
)
|
||||
|
||||
result = await provider.transcribe(
|
||||
@@ -134,7 +144,7 @@ class TestOpenRouterProviderTranscribe:
|
||||
"""Transcribe converts SDK failures to ProviderError."""
|
||||
provider = OpenRouterTranscriptionProvider(
|
||||
settings=Settings(openrouter_api_key="test-key"),
|
||||
client=_FakeClient(error=RuntimeError("network down")),
|
||||
client=_fake_client(error=RuntimeError("network down")),
|
||||
)
|
||||
|
||||
with pytest.raises(ProviderError):
|
||||
@@ -149,7 +159,7 @@ class TestOpenRouterProviderTranscribe:
|
||||
"""Caller and shutdown cancellation must not be relabeled as a timeout."""
|
||||
provider = OpenRouterTranscriptionProvider(
|
||||
settings=Settings(openrouter_api_key="test-key"),
|
||||
client=_FakeClient(error=asyncio.CancelledError()),
|
||||
client=_fake_client(error=asyncio.CancelledError()),
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
@@ -166,7 +176,7 @@ class TestOpenRouterProviderTranscribe:
|
||||
client = _FakeClient(response=response)
|
||||
provider = OpenRouterTranscriptionProvider(
|
||||
settings=Settings(openrouter_api_key="test-key"),
|
||||
client=client,
|
||||
client=cast("OpenRouter", client),
|
||||
)
|
||||
|
||||
await provider.transcribe(
|
||||
@@ -185,7 +195,7 @@ class TestOpenRouterProviderTranscribe:
|
||||
"""Transcribe raises ProviderResponseError for missing completion text."""
|
||||
provider = OpenRouterTranscriptionProvider(
|
||||
settings=Settings(openrouter_api_key="test-key"),
|
||||
client=_FakeClient(response=SimpleNamespace(choices=[])),
|
||||
client=_fake_client(response=SimpleNamespace(choices=[])),
|
||||
)
|
||||
|
||||
with pytest.raises(ProviderResponseError):
|
||||
@@ -204,7 +214,7 @@ class TestOpenRouterProviderTranscribe:
|
||||
}
|
||||
provider = OpenRouterTranscriptionProvider(
|
||||
settings=Settings(openrouter_api_key="test-key"),
|
||||
client=_FakeClient(response=response),
|
||||
client=_fake_client(response=response),
|
||||
)
|
||||
|
||||
result = await provider.transcribe(
|
||||
|
||||
@@ -4,6 +4,7 @@ from datetime import timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import event
|
||||
|
||||
from transcription.db.models import Document
|
||||
from transcription.db.models import Job
|
||||
@@ -101,7 +102,7 @@ class TestJobService:
|
||||
assert result[0].id == job.id
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_next_queued_job_orders_by_created_date(
|
||||
async def test_claim_next_queued_job_claims_oldest_and_marks_processing(
|
||||
self,
|
||||
job_service: JobService,
|
||||
document_service: DocumentService,
|
||||
@@ -119,9 +120,42 @@ class TestJobService:
|
||||
await job_service.create_job(job=first)
|
||||
await job_service.create_job(job=second)
|
||||
|
||||
next_job = await job_service.read_next_queued_job()
|
||||
assert next_job is not None
|
||||
assert next_job.id == first.id
|
||||
claimed = await job_service.claim_next_queued_job()
|
||||
assert claimed is not None
|
||||
assert claimed.id == first.id
|
||||
assert claimed.status == JobStatus.PROCESSING
|
||||
|
||||
# The claim is exclusive: the same job is never handed out twice.
|
||||
next_claim = await job_service.claim_next_queued_job()
|
||||
assert next_claim is not None
|
||||
assert next_claim.id == second.id
|
||||
|
||||
assert await job_service.claim_next_queued_job() is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claim_next_queued_job_emits_a_bounded_unadorned_query(
|
||||
self,
|
||||
job_service: JobService,
|
||||
):
|
||||
"""CRIT-01: the hot poll must not select a subgraph or scan the queue."""
|
||||
statements: list[str] = []
|
||||
|
||||
async with job_service._session_scope() as session:
|
||||
bind = session.get_bind()
|
||||
|
||||
def capture(_conn, _cursor, statement, *_rest):
|
||||
statements.append(statement)
|
||||
|
||||
event.listen(bind, "before_cursor_execute", capture)
|
||||
try:
|
||||
await job_service.claim_next_queued_job(session=session)
|
||||
finally:
|
||||
event.remove(bind, "before_cursor_execute", capture)
|
||||
|
||||
selects = [item for item in statements if item.lstrip().upper().startswith("SELECT")]
|
||||
assert len(selects) == 1, selects
|
||||
assert "LIMIT" in selects[0].upper()
|
||||
assert "JOIN" not in selects[0].upper()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_job_persists_provider_and_model(
|
||||
|
||||
@@ -14,11 +14,13 @@ from transcription.db.models import JobSource
|
||||
from transcription.db.models import Source
|
||||
from transcription.providers import RequestManifest
|
||||
from transcription.providers import TranscriptionResult
|
||||
from transcription.providers.evidence import SourceEvidenceReference
|
||||
from transcription.providers.evidence import build_software_context
|
||||
from transcription.services import ServiceBundle
|
||||
from transcription.services.documents import DocumentService
|
||||
from transcription.services.jobs import JobService
|
||||
from transcription.services.normalization import normalize_orientation
|
||||
from transcription.services.normalization import normalize_orientation_async
|
||||
from transcription.services.sources import SourceService
|
||||
from transcription.services.workflows import process_queued_job
|
||||
|
||||
@@ -61,7 +63,9 @@ def test_orientation_three_is_physically_rotated_and_metadata_removed(tmp_path):
|
||||
with Image.open(path) as source_image, Image.open(io.BytesIO(result.content)) as derivative:
|
||||
assert source_image.getexif()[274] == 3
|
||||
assert derivative.getexif().get(274, 1) == 1
|
||||
assert derivative.getpixel((0, 0))[2] > derivative.getpixel((0, 0))[0]
|
||||
pixel = derivative.getpixel((0, 0))
|
||||
assert isinstance(pixel, tuple)
|
||||
assert pixel[2] > pixel[0]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -129,7 +133,9 @@ async def test_resolve_provider_input_persists_exact_derivative(default_session_
|
||||
assert provider_input.derivative_id == artifacts[0].id
|
||||
assert provider_input.path.read_bytes() != original
|
||||
assert hashlib.sha256(provider_input.path.read_bytes()).hexdigest() == provider_input.digest_sha256
|
||||
assert artifacts[0].coordinate_metadata["original_orientation"] == 3
|
||||
coordinate_metadata = artifacts[0].coordinate_metadata
|
||||
assert coordinate_metadata is not None
|
||||
assert coordinate_metadata["original_orientation"] == 3
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@@ -218,11 +224,29 @@ async def test_worker_sends_exact_derivative_and_links_attempt_evidence(
|
||||
attempts = await services.sources.list_execution_attempts(source_id=source.id)
|
||||
artifacts = await services.sources.list_processing_artifacts(source_id=source.id)
|
||||
source_reference = captured["source_reference"]
|
||||
assert isinstance(source_reference, SourceEvidenceReference)
|
||||
captured_bytes = captured["bytes"]
|
||||
assert isinstance(captured_bytes, bytes)
|
||||
assert source_path.read_bytes() == original
|
||||
assert hashlib.sha256(captured["bytes"]).hexdigest() == source_reference.digest_sha256
|
||||
assert hashlib.sha256(captured_bytes).hexdigest() == source_reference.digest_sha256
|
||||
assert source_reference.derivative_id is not None
|
||||
assert {artifact.artifact_type for artifact in artifacts} == {
|
||||
"orientation_normalized_model_input",
|
||||
"transcription_quality_warnings",
|
||||
}
|
||||
assert {artifact.execution_attempt_id for artifact in artifacts} == {attempts[0].id}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_wrapper_matches_sync_result_and_precomputes_digest(tmp_path):
|
||||
"""[MED-01]: Pillow work runs off the event loop and hashes its own output."""
|
||||
path = tmp_path / "async-upside-down.jpg"
|
||||
_write_oriented_jpeg(path, orientation=3)
|
||||
|
||||
result = await normalize_orientation_async(path, media_type="image/jpeg")
|
||||
expected = normalize_orientation(path, media_type="image/jpeg")
|
||||
|
||||
assert result is not None
|
||||
assert expected is not None
|
||||
assert result.content == expected.content
|
||||
assert result.digest_sha256 == hashlib.sha256(result.content).hexdigest()
|
||||
|
||||
@@ -2,6 +2,7 @@ from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlmodel import col
|
||||
from sqlmodel import select
|
||||
|
||||
from transcription.config import Settings
|
||||
@@ -63,7 +64,7 @@ async def test_create_job_for_document_sorts_sources_and_creates_links(async_ses
|
||||
|
||||
sources = (
|
||||
await async_session.exec(
|
||||
select(Source).where(Source.document_id == document.id).order_by(Source.page_number) # pyright: ignore[reportArgumentType]
|
||||
select(Source).where(Source.document_id == document.id).order_by(col(Source.page_number))
|
||||
)
|
||||
).all()
|
||||
assert [source.upload_name for source in sources] == ["A_page.pdf", "b_page.pdf"]
|
||||
@@ -99,7 +100,7 @@ async def test_create_document_job_stores_source_under_document_id_directory(asy
|
||||
|
||||
source = (
|
||||
await async_session.exec(
|
||||
select(Source).where(Source.document_id == result.document_id).order_by(Source.page_number) # pyright: ignore[reportArgumentType]
|
||||
select(Source).where(Source.document_id == result.document_id).order_by(col(Source.page_number))
|
||||
)
|
||||
).first()
|
||||
assert source is not None
|
||||
@@ -114,11 +115,12 @@ async def test_create_document_job_stores_source_under_document_id_directory(asy
|
||||
assert created_job.prompt_name == "transcribe_document.md"
|
||||
|
||||
|
||||
def test_store_person_portrait_stores_file_under_person_id_directory(tmp_path):
|
||||
@pytest.mark.asyncio
|
||||
async def test_store_person_portrait_stores_file_under_person_id_directory(tmp_path):
|
||||
settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
|
||||
person_id = uuid4()
|
||||
|
||||
stored_path = store_person_portrait(
|
||||
stored_path = await store_person_portrait(
|
||||
person_id=person_id,
|
||||
filename="portrait.png",
|
||||
file_bytes=b"portrait-bytes",
|
||||
@@ -144,8 +146,9 @@ def test_source_mime_type_uses_canonical_source_policy(filename, expected_mime_t
|
||||
assert source_mime_type(filename) == expected_mime_type
|
||||
|
||||
|
||||
def test_source_storage_rejects_unsupported_format(tmp_path):
|
||||
@pytest.mark.asyncio
|
||||
async def test_source_storage_rejects_unsupported_format(tmp_path):
|
||||
settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
|
||||
|
||||
with pytest.raises(SourceStorageError):
|
||||
store_source_file(filename="page.txt", file_bytes=b"text", settings=settings)
|
||||
await store_source_file(filename="page.txt", file_bytes=b"text", settings=settings)
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Phase 6 verification that modification timestamps advance automatically.
|
||||
|
||||
Covers the `onupdate` change: `updated_at` / `date_updated` are now maintained
|
||||
by the ORM column default rather than by hand at each call site, so update paths
|
||||
that previously forgot to set them no longer report a stale timestamp.
|
||||
"""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from transcription.db.models import Document
|
||||
from transcription.db.models import Job
|
||||
from transcription.db.models import JobStatus
|
||||
from transcription.db.models import Person
|
||||
from transcription.services.documents import DocumentService
|
||||
from transcription.services.jobs import JobService
|
||||
from transcription.services.people import PeopleService
|
||||
from transcription.services.people import PersonRoleRegistry
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_document_update_advances_updated_at(default_session_factory):
|
||||
documents = DocumentService(session_factory=default_session_factory)
|
||||
document = await documents.create_document(Document(id=uuid4(), name="timestamps"))
|
||||
original = document.updated_at
|
||||
|
||||
document.name = "timestamps renamed"
|
||||
updated = await documents.update_document(document)
|
||||
|
||||
assert updated.updated_at > original
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_person_update_advances_updated_at(default_session_factory):
|
||||
people = PeopleService(session_factory=default_session_factory)
|
||||
person = await people.create_person(Person(full_name="Grace Hopper"))
|
||||
original = person.updated_at
|
||||
|
||||
person.full_name = "Rear Adm. Grace Hopper"
|
||||
updated = await people.update_person(person)
|
||||
|
||||
assert updated.updated_at > original
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_update_advances_updated_at(default_session_factory):
|
||||
roles = PersonRoleRegistry(session_factory=default_session_factory)
|
||||
role = await roles.create_entry(label="Witness")
|
||||
original = role.updated_at
|
||||
|
||||
updated = await roles.update_entry(role.id, label="Chief Witness", is_active=True)
|
||||
|
||||
assert updated.updated_at > original
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_job_status_update_advances_date_updated(default_session_factory):
|
||||
documents = DocumentService(session_factory=default_session_factory)
|
||||
jobs = JobService(session_factory=default_session_factory)
|
||||
|
||||
document = await documents.create_document(Document(id=uuid4(), name="job-timestamps"))
|
||||
job = await jobs.create_job(Job(document_id=document.id))
|
||||
original = job.date_updated
|
||||
|
||||
updated = await jobs.update_job_state(job_id=job.id, status=JobStatus.PROCESSING)
|
||||
|
||||
assert updated.date_updated > original
|
||||
@@ -5,7 +5,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from transcription.services.transcription import transcribe_document_image
|
||||
from transcription.services.sources import transcribe_document_image
|
||||
|
||||
HAS_OPENROUTER_KEY = bool(os.getenv("OPENROUTER_API_KEY"))
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import event
|
||||
|
||||
from transcription.config import Settings
|
||||
from transcription.db.models import Document
|
||||
@@ -232,3 +233,112 @@ class TestSourceServiceRevisionUpsert:
|
||||
|
||||
with pytest.raises(SourceDeleteBlockedError):
|
||||
await transcriptions.delete_unlinked_source(source_id=source.id)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestSourceServiceQueryShape:
|
||||
"""LOW-08: reads must filter and bound in SQL, not in Python."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_sources_detail_filters_job_id_with_a_join(self, default_session_factory):
|
||||
documents = DocumentService(session_factory=default_session_factory)
|
||||
jobs = JobService(session_factory=default_session_factory)
|
||||
transcriptions = SourceService(session_factory=default_session_factory)
|
||||
|
||||
document = Document(id=uuid4(), name="join-filter")
|
||||
await documents.create_document(document=document)
|
||||
job = Job(document_id=document.id, status=JobStatus.QUEUED)
|
||||
other_job = Job(document_id=document.id, status=JobStatus.QUEUED)
|
||||
await jobs.create_job(job=job)
|
||||
await jobs.create_job(job=other_job)
|
||||
|
||||
linked = Source(
|
||||
document_id=document.id,
|
||||
page_number=1,
|
||||
upload_name="linked.jpg",
|
||||
filename="linked.jpg",
|
||||
file_path="uploads/linked.jpg",
|
||||
file_hash="7" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
unlinked = Source(
|
||||
document_id=document.id,
|
||||
page_number=2,
|
||||
upload_name="unlinked.jpg",
|
||||
filename="unlinked.jpg",
|
||||
file_path="uploads/unlinked.jpg",
|
||||
file_hash="8" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
async with transcriptions._session_scope() as session:
|
||||
session.add_all((linked, unlinked))
|
||||
await session.flush()
|
||||
session.add(JobSource(job_id=job.id, source_id=linked.id, status=JobSourceStatus.PENDING))
|
||||
session.add(JobSource(job_id=other_job.id, source_id=unlinked.id, status=JobSourceStatus.PENDING))
|
||||
await session.commit()
|
||||
|
||||
statements: list[str] = []
|
||||
|
||||
async with transcriptions._session_scope() as session:
|
||||
bind = session.get_bind()
|
||||
|
||||
def capture(_conn, _cursor, statement, *_rest):
|
||||
statements.append(statement)
|
||||
|
||||
event.listen(bind, "before_cursor_execute", capture)
|
||||
try:
|
||||
sources = await transcriptions.list_sources_detail(job_id=job.id, session=session)
|
||||
finally:
|
||||
event.remove(bind, "before_cursor_execute", capture)
|
||||
|
||||
assert [source.id for source in sources] == [linked.id]
|
||||
|
||||
primary = next(item for item in statements if item.lstrip().upper().startswith("SELECT"))
|
||||
assert "JOIN" in primary.upper()
|
||||
assert "JOBSOURCE" in primary.upper().replace("_", "")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_source_navigation_does_not_scan_every_sibling(self, default_session_factory):
|
||||
documents = DocumentService(session_factory=default_session_factory)
|
||||
transcriptions = SourceService(session_factory=default_session_factory)
|
||||
|
||||
document = Document(id=uuid4(), name="navigation-bounds")
|
||||
await documents.create_document(document=document)
|
||||
|
||||
pages = [
|
||||
Source(
|
||||
document_id=document.id,
|
||||
page_number=page_number,
|
||||
upload_name=f"page-{page_number}.jpg",
|
||||
filename=f"page-{page_number}.jpg",
|
||||
file_path=f"uploads/page-{page_number}.jpg",
|
||||
file_hash=str(page_number) * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
for page_number in range(1, 5)
|
||||
]
|
||||
async with transcriptions._session_scope() as session:
|
||||
session.add_all(pages)
|
||||
await session.commit()
|
||||
for page in pages:
|
||||
await session.refresh(page)
|
||||
|
||||
statements: list[str] = []
|
||||
|
||||
async with transcriptions._session_scope() as session:
|
||||
bind = session.get_bind()
|
||||
|
||||
def capture(_conn, _cursor, statement, *_rest):
|
||||
statements.append(statement)
|
||||
|
||||
event.listen(bind, "before_cursor_execute", capture)
|
||||
try:
|
||||
navigation = await transcriptions.read_source_navigation(pages[1].id, session=session)
|
||||
finally:
|
||||
event.remove(bind, "before_cursor_execute", capture)
|
||||
|
||||
assert navigation.previous_id == pages[0].id
|
||||
assert navigation.next_id == pages[2].id
|
||||
|
||||
adjacency = [item for item in statements if "LIMIT" in item.upper()]
|
||||
assert len(adjacency) == 2, statements
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from pydantic import JsonValue
|
||||
|
||||
from transcription.db.models import Document
|
||||
from transcription.db.models import DocumentPerson
|
||||
@@ -303,8 +304,11 @@ async def test_update_job_source_transcription_persists_provider_json_payloads(d
|
||||
JobSource(job_id=job.id, source_id=source.id, status=JobSourceStatus.PENDING)
|
||||
)
|
||||
|
||||
metadata = {"finish_reason": "stop", "usage": {"input_tokens": 11, "output_tokens": 22, "total_tokens": 33}}
|
||||
raw_payload = {"id": "resp_xyz", "choices": [{"message": {"content": "provider transcript"}}]}
|
||||
metadata: dict[str, JsonValue] = {
|
||||
"finish_reason": "stop",
|
||||
"usage": {"input_tokens": 11, "output_tokens": 22, "total_tokens": 33},
|
||||
}
|
||||
raw_payload: dict[str, JsonValue] = {"id": "resp_xyz", "choices": [{"message": {"content": "provider transcript"}}]}
|
||||
|
||||
await transcriptions.update_job_source_transcription(
|
||||
job_id=job.id,
|
||||
|
||||
@@ -20,6 +20,7 @@ class TestAppFactory:
|
||||
app = create_app()
|
||||
assert isinstance(app, FastAPI)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
class TestAppLifespan:
|
||||
"""Verify startup and shutdown lifecycle behavior."""
|
||||
|
||||
+30
-4
@@ -1,6 +1,7 @@
|
||||
"""Tests for transcription.config — settings loading, provider defaults, and paths."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
@@ -10,9 +11,9 @@ from transcription.config import Settings
|
||||
from transcription.config import parse_cli_settings
|
||||
|
||||
|
||||
def _make_settings(**overrides) -> Settings:
|
||||
def _make_settings(**overrides: Any) -> Settings:
|
||||
"""Build a Settings instance with a dummy API key unless overridden."""
|
||||
defaults = {"openrouter_api_key": "test-key-abc123", "provider_models": None}
|
||||
defaults: dict[str, Any] = {"openrouter_api_key": "test-key-abc123", "provider_models": None}
|
||||
defaults.update(overrides)
|
||||
return Settings(**defaults)
|
||||
|
||||
@@ -146,7 +147,32 @@ class TestWorkerReliabilitySettings:
|
||||
"""Verify worker retry settings defaults."""
|
||||
|
||||
def test_worker_retry_defaults(self):
|
||||
"""worker retry settings default to no retries and no backoff."""
|
||||
"""worker retry settings default to no retries."""
|
||||
settings = _make_settings()
|
||||
assert settings.worker_max_retries == 0
|
||||
assert settings.worker_retry_backoff_seconds == 0.0
|
||||
|
||||
|
||||
def test_provider_timeout_is_not_capped_at_twenty_seconds():
|
||||
"""HIGH-03: vision transcription regularly runs past the old le=20.0 ceiling."""
|
||||
settings = Settings(openrouter_api_key="test-key", worker_provider_timeout_seconds=300.0)
|
||||
assert settings.worker_provider_timeout_seconds == 300.0
|
||||
|
||||
|
||||
def test_provider_timeout_must_still_be_positive():
|
||||
with pytest.raises(ValidationError):
|
||||
Settings(openrouter_api_key="test-key", worker_provider_timeout_seconds=0.0)
|
||||
|
||||
|
||||
def test_openrouter_client_timeout_tracks_the_configured_budget():
|
||||
"""HIGH-03: httpx defaults every phase to 5s, silently capping the provider call."""
|
||||
from transcription.providers.openrouter import OpenRouterTranscriptionProvider
|
||||
|
||||
settings = Settings(openrouter_api_key="test-key", worker_provider_timeout_seconds=123.0)
|
||||
provider = OpenRouterTranscriptionProvider(settings=settings)
|
||||
assert provider._capturing_client is not None
|
||||
timeout = provider._capturing_client._client.timeout
|
||||
|
||||
assert timeout.read == 123.0
|
||||
assert timeout.write == 123.0
|
||||
assert timeout.pool == 123.0
|
||||
assert timeout.connect == 10.0
|
||||
|
||||
+45
-71
@@ -1,8 +1,13 @@
|
||||
"""Tests for the database runtime and V2 schema bootstrap behavior."""
|
||||
|
||||
import warnings
|
||||
|
||||
import pytest
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.dialects import postgresql
|
||||
from sqlalchemy.exc import SAWarning
|
||||
from sqlmodel import SQLModel
|
||||
from sqlmodel import select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
@@ -12,9 +17,9 @@ from transcription.db import create_all
|
||||
from transcription.db import dispose_database_runtime
|
||||
from transcription.db import initialize_database_runtime
|
||||
from transcription.db import session_scope
|
||||
from transcription.db import upgrade_schema
|
||||
from transcription.db.models import DocumentType
|
||||
from transcription.db.models import PersonRole
|
||||
from transcription.db.models import Source
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -106,92 +111,61 @@ async def test_create_all_seeds_default_registry_rows(tmp_path):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_all_upgrades_existing_person_table_for_family_search(tmp_path):
|
||||
async def test_create_all_declares_hot_path_indexes(tmp_path):
|
||||
"""Worker and detail-page filters must be index-backed in a freshly created schema."""
|
||||
settings = Settings(
|
||||
openrouter_api_key="test-key",
|
||||
database=SqliteSettings(path=str(tmp_path / "upgrade.db")),
|
||||
database=SqliteSettings(path=str(tmp_path / "indexes.db")),
|
||||
environment="test",
|
||||
)
|
||||
runtime = initialize_database_runtime(settings=settings)
|
||||
|
||||
try:
|
||||
async with runtime.engine.begin() as connection:
|
||||
await connection.execute(
|
||||
text("CREATE TABLE person (id CHAR(32) PRIMARY KEY NOT NULL, full_name VARCHAR NOT NULL)")
|
||||
)
|
||||
|
||||
await create_all(engine=runtime.engine)
|
||||
async with runtime.engine.connect() as connection:
|
||||
columns, indexes = await connection.run_sync(
|
||||
lambda sync_connection: (
|
||||
{column["name"] for column in inspect(sync_connection).get_columns("person")},
|
||||
inspect(sync_connection).get_indexes("person"),
|
||||
)
|
||||
)
|
||||
|
||||
assert "family_search_id" in columns
|
||||
assert any(index["column_names"] == ["family_search_id"] and index["unique"] for index in indexes)
|
||||
def collect(sync_connection) -> dict[str, list[list[str]]]:
|
||||
database = inspect(sync_connection)
|
||||
return {
|
||||
table: [index["column_names"] for index in database.get_indexes(table)]
|
||||
for table in ("job", "source", "job_source", "document", "document_person")
|
||||
}
|
||||
|
||||
indexes = await connection.run_sync(collect)
|
||||
|
||||
assert ["status", "date_created"] in indexes["job"]
|
||||
assert ["document_id"] in indexes["job"]
|
||||
assert ["document_id"] in indexes["source"]
|
||||
assert ["preferred_execution_attempt_id"] in indexes["source"]
|
||||
assert ["job_id"] in indexes["job_source"]
|
||||
assert ["source_id"] in indexes["job_source"]
|
||||
assert ["document_type_id"] in indexes["document"]
|
||||
for column in ("document_id", "person_id", "role_id"):
|
||||
assert [column] in indexes["document_person"]
|
||||
finally:
|
||||
await dispose_database_runtime()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v42_upgrade_adds_evidence_tables_without_rewriting_legacy_snapshot(tmp_path):
|
||||
settings = Settings(
|
||||
openrouter_api_key="test-key",
|
||||
database=SqliteSettings(path=str(tmp_path / "v42-upgrade.db")),
|
||||
environment="test",
|
||||
)
|
||||
runtime = initialize_database_runtime(settings=settings)
|
||||
def test_metadata_has_no_unresolvable_table_cycle():
|
||||
"""create_all must be able to order every table, including on PostgreSQL."""
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", SAWarning)
|
||||
ordered = [table.name for table in SQLModel.metadata.sorted_tables]
|
||||
|
||||
try:
|
||||
async with runtime.engine.begin() as connection:
|
||||
await connection.execute(text("CREATE TABLE job (id CHAR(32) PRIMARY KEY NOT NULL)"))
|
||||
await connection.execute(text("CREATE TABLE source (id CHAR(32) PRIMARY KEY NOT NULL)"))
|
||||
await connection.execute(
|
||||
text(
|
||||
"CREATE TABLE job_source ("
|
||||
"id CHAR(32) PRIMARY KEY NOT NULL, "
|
||||
"job_id CHAR(32) NOT NULL, "
|
||||
"source_id CHAR(32) NOT NULL, "
|
||||
"raw_api_response JSON"
|
||||
")"
|
||||
)
|
||||
)
|
||||
await connection.execute(text("INSERT INTO job (id) VALUES ('job-1')"))
|
||||
await connection.execute(text("INSERT INTO source (id) VALUES ('source-1')"))
|
||||
await connection.execute(
|
||||
text(
|
||||
"INSERT INTO job_source (id, job_id, source_id, raw_api_response) "
|
||||
"VALUES ('link-1', 'job-1', 'source-1', :snapshot)"
|
||||
),
|
||||
{"snapshot": '{"legacy":true}'},
|
||||
)
|
||||
assert ordered.index("source") < ordered.index("execution_attempt")
|
||||
|
||||
await upgrade_schema(engine=runtime.engine)
|
||||
await upgrade_schema(engine=runtime.engine)
|
||||
async with runtime.engine.connect() as connection:
|
||||
table_names = set(await connection.run_sync(lambda c: inspect(c).get_table_names()))
|
||||
job_columns = set(
|
||||
await connection.run_sync(
|
||||
lambda c: tuple(column["name"] for column in inspect(c).get_columns("job"))
|
||||
)
|
||||
)
|
||||
source_columns = set(
|
||||
await connection.run_sync(
|
||||
lambda c: tuple(column["name"] for column in inspect(c).get_columns("source"))
|
||||
)
|
||||
)
|
||||
legacy_snapshot = (
|
||||
await connection.execute(text("SELECT raw_api_response FROM job_source WHERE id = 'link-1'"))
|
||||
).scalar_one()
|
||||
|
||||
assert {"execution_attempt", "processing_artifact"}.issubset(table_names)
|
||||
assert "purpose" in job_columns
|
||||
assert "preferred_execution_attempt_id" in source_columns
|
||||
assert "legacy" in legacy_snapshot
|
||||
finally:
|
||||
await dispose_database_runtime()
|
||||
def test_preferred_execution_attempt_id_column_matches_model_declaration():
|
||||
"""The self-referential provenance column is a real UUID, not an opaque CHAR(32)."""
|
||||
column = SQLModel.metadata.tables["source"].c["preferred_execution_attempt_id"]
|
||||
|
||||
assert isinstance(column.type, sa.Uuid)
|
||||
assert column.type.compile(dialect=postgresql.dialect()) == "UUID"
|
||||
assert Source.model_fields["preferred_execution_attempt_id"].annotation is not None
|
||||
|
||||
foreign_key = next(iter(column.foreign_keys))
|
||||
assert foreign_key.use_alter is True
|
||||
assert foreign_key.column is SQLModel.metadata.tables["execution_attempt"].c["id"]
|
||||
|
||||
|
||||
def test_bootstrap_policy_production_defaults_false():
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Phase 6 verification for the URL-keyed engine and session-factory registries.
|
||||
|
||||
Covers [MED-04]: replacing `functools.cache` with an explicit registry so that
|
||||
disposing one database's engine cannot silently tear down every other one.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from transcription.db.engine import dispose_engine
|
||||
from transcription.db.engine import get_engine
|
||||
from transcription.db.session import dispose_session_factory
|
||||
from transcription.db.session import get_session_factory
|
||||
|
||||
URL_A = "sqlite+aiosqlite:///./.registry-test-a.db"
|
||||
URL_B = "sqlite+aiosqlite:///./.registry-test-b.db"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_distinct_urls_produce_distinct_engines_and_eviction_is_targeted():
|
||||
engine_a = get_engine(URL_A)
|
||||
engine_b = get_engine(URL_B)
|
||||
|
||||
assert engine_a is not engine_b
|
||||
assert get_engine(URL_A) is engine_a
|
||||
|
||||
await dispose_engine(URL_A)
|
||||
|
||||
assert get_engine(URL_B) is engine_b, "disposing one URL must not evict the others"
|
||||
assert get_engine(URL_A) is not engine_a, "the disposed URL must be rebuilt on demand"
|
||||
|
||||
await dispose_engine(URL_A)
|
||||
await dispose_engine(URL_B)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disposing_an_unregistered_url_is_a_noop():
|
||||
await dispose_engine("sqlite+aiosqlite:///./.registry-test-never-created.db")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_factory_eviction_is_targeted():
|
||||
factory_a = get_session_factory(URL_A)
|
||||
factory_b = get_session_factory(URL_B)
|
||||
|
||||
assert factory_a is not factory_b
|
||||
|
||||
await dispose_session_factory(URL_A)
|
||||
|
||||
assert get_session_factory(URL_B) is factory_b
|
||||
assert get_session_factory(URL_A) is not factory_a
|
||||
|
||||
await dispose_session_factory(URL_A)
|
||||
await dispose_session_factory(URL_B)
|
||||
+23
-9
@@ -1,5 +1,6 @@
|
||||
"""Tests for the V2 SQLModel persistence layer and relationships."""
|
||||
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
@@ -17,8 +18,8 @@ from transcription.db.models import PersonRole
|
||||
from transcription.db.models import Source
|
||||
|
||||
|
||||
def _make_document(**overrides) -> Document:
|
||||
defaults = {
|
||||
def _make_document(**overrides: Any) -> Document:
|
||||
defaults: dict[str, Any] = {
|
||||
"name": "letter bundle",
|
||||
"notes": "Family correspondence",
|
||||
}
|
||||
@@ -51,8 +52,8 @@ def _persist_document(session) -> Document:
|
||||
return document
|
||||
|
||||
|
||||
def _persist_person(session, **overrides) -> Person:
|
||||
defaults = {"full_name": "Ada Lovelace"}
|
||||
def _persist_person(session, **overrides: Any) -> Person:
|
||||
defaults: dict[str, Any] = {"full_name": "Ada Lovelace"}
|
||||
defaults.update(overrides)
|
||||
person = Person(**defaults)
|
||||
session.add(person)
|
||||
@@ -69,8 +70,8 @@ def _persist_job(session, document: Document) -> Job:
|
||||
return job
|
||||
|
||||
|
||||
def _persist_source(session, document: Document, *, page_number: int = 1, **overrides) -> Source:
|
||||
defaults = {
|
||||
def _persist_source(session, document: Document, *, page_number: int = 1, **overrides: Any) -> Source:
|
||||
defaults: dict[str, Any] = {
|
||||
"document_id": document.id,
|
||||
"page_number": page_number,
|
||||
"upload_name": "letter.jpg",
|
||||
@@ -88,8 +89,8 @@ def _persist_source(session, document: Document, *, page_number: int = 1, **over
|
||||
return source
|
||||
|
||||
|
||||
def _persist_job_source(session, job: Job, source: Source, **overrides) -> JobSource:
|
||||
defaults = {
|
||||
def _persist_job_source(session, job: Job, source: Source, **overrides: Any) -> JobSource:
|
||||
defaults: dict[str, Any] = {
|
||||
"job_id": job.id,
|
||||
"source_id": source.id,
|
||||
"status": JobSourceStatus.PENDING,
|
||||
@@ -246,7 +247,7 @@ class TestRelationships:
|
||||
session.add(link)
|
||||
session.commit()
|
||||
|
||||
session.refresh(document)
|
||||
session.refresh(document, attribute_names=["jobs", "sources", "document_people"])
|
||||
assert len(document.jobs) == 1
|
||||
assert len(document.sources) == 1
|
||||
assert len(document.document_people) == 1
|
||||
@@ -266,3 +267,16 @@ class TestRegistryModels:
|
||||
session.add(duplicate)
|
||||
with pytest.raises(IntegrityError):
|
||||
session.commit()
|
||||
|
||||
|
||||
def test_no_relationship_declares_an_implicit_eager_load():
|
||||
"""CRIT-02 guard: eager loading is a per-query decision, never a model default."""
|
||||
from sqlmodel import SQLModel
|
||||
|
||||
offenders = {
|
||||
f"{mapper.class_.__name__}.{relationship.key}": relationship.lazy
|
||||
for mapper in SQLModel._sa_registry.mappers
|
||||
for relationship in mapper.relationships
|
||||
if relationship.lazy not in {"raise", "noload"}
|
||||
}
|
||||
assert offenders == {}, f"Relationships must not preload by default: {offenders}"
|
||||
|
||||
@@ -86,7 +86,7 @@ class TestPromptConfiguration:
|
||||
assert execution.temperature == 0.2
|
||||
assert execution.top_p == 0.9
|
||||
with pytest.raises(ValidationError):
|
||||
execution.prompt_name = "changed.md"
|
||||
execution.prompt_name = "changed.md" # ty: ignore[invalid-assignment]
|
||||
|
||||
def test_rejects_prompt_path_traversal_even_with_direct_loader_call(self, tmp_path):
|
||||
outside_prompt = tmp_path / "outside.md"
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Structural rules for the services package.
|
||||
|
||||
`.github/instructions/services.instructions.md:13` requires that service classes
|
||||
stay independent of one another. Shared behavior belongs in a neutral module
|
||||
(`base.py`, `registry.py`, `source_media.py`, `media_storage.py`), and any
|
||||
operation spanning two services belongs in an orchestration module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
SERVICES_DIR = Path(__file__).resolve().parents[1] / "src" / "transcription" / "services"
|
||||
|
||||
# Modules that intentionally compose several services rather than owning one table.
|
||||
ORCHESTRATION_MODULES = frozenset({"store", "workflows", "__init__"})
|
||||
|
||||
|
||||
def _module_paths() -> list[Path]:
|
||||
return sorted(SERVICES_DIR.glob("*.py"))
|
||||
|
||||
|
||||
def _defines_service_class(tree: ast.Module) -> bool:
|
||||
return any(
|
||||
isinstance(node, ast.ClassDef) and node.name.endswith("Service") and node.name != "RegistryService"
|
||||
for node in tree.body
|
||||
)
|
||||
|
||||
|
||||
def _service_modules() -> dict[str, ast.Module]:
|
||||
modules: dict[str, ast.Module] = {}
|
||||
for path in _module_paths():
|
||||
if path.stem in ORCHESTRATION_MODULES:
|
||||
continue
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"))
|
||||
if _defines_service_class(tree):
|
||||
modules[path.stem] = tree
|
||||
return modules
|
||||
|
||||
|
||||
def _imported_sibling_modules(tree: ast.Module) -> set[str]:
|
||||
imported: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node.level == 1 and node.module:
|
||||
imported.add(node.module.split(".")[0])
|
||||
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
|
||||
parts = node.module.split(".")
|
||||
if parts[:2] == ["transcription", "services"] and len(parts) > 2:
|
||||
imported.add(parts[2])
|
||||
return imported
|
||||
|
||||
|
||||
def test_service_modules_are_discovered():
|
||||
"""Guard the guard: the rule below is meaningless if nothing is scanned."""
|
||||
assert set(_service_modules()) >= {"documents", "jobs", "people", "sources"}
|
||||
|
||||
|
||||
def test_no_service_module_imports_another_service_module():
|
||||
"""MED-14: a service module must not depend on a sibling service module."""
|
||||
modules = _service_modules()
|
||||
violations = {
|
||||
name: sorted(_imported_sibling_modules(tree) & set(modules) - {name}) for name, tree in modules.items()
|
||||
}
|
||||
assert {name: found for name, found in violations.items() if found} == {}
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Structural rules for the NiceGUI page layer.
|
||||
|
||||
`.github/instructions/ui.instructions.md:23,31` forbids pages from owning
|
||||
persistence or query-building concerns, and forbids components from resolving
|
||||
request or application state. HIGH-07 recorded three violations of those rules;
|
||||
these tests keep them from coming back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
UI_DIR = Path(__file__).resolve().parents[1] / "src" / "transcription" / "ui"
|
||||
PAGES_DIR = UI_DIR / "pages"
|
||||
COMPONENTS_DIR = UI_DIR / "components"
|
||||
|
||||
# Names a page must not pull in: they hand the page a session, a transaction, ORM
|
||||
# loader introspection, or process-global configuration.
|
||||
FORBIDDEN_PAGE_IMPORTS = frozenset(
|
||||
{
|
||||
"session_scope",
|
||||
"transaction_scope",
|
||||
"get_session_factory",
|
||||
"resolve_session_factory",
|
||||
"get_settings",
|
||||
"get_engine",
|
||||
"upgrade_schema",
|
||||
"create_all",
|
||||
}
|
||||
)
|
||||
|
||||
FORBIDDEN_PAGE_MODULES = frozenset({"sqlalchemy", "sqlmodel"})
|
||||
|
||||
|
||||
def _page_paths() -> list[Path]:
|
||||
return sorted(path for path in PAGES_DIR.glob("*.py") if path.stem != "__init__")
|
||||
|
||||
|
||||
def _component_paths() -> list[Path]:
|
||||
return sorted(COMPONENTS_DIR.rglob("*.py"))
|
||||
|
||||
|
||||
def _imports(tree: ast.Module) -> tuple[set[str], set[str]]:
|
||||
"""Return (imported names, imported root modules)."""
|
||||
names: set[str] = set()
|
||||
modules: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
if node.module and node.level == 0:
|
||||
modules.add(node.module.split(".")[0])
|
||||
names.update(alias.name for alias in node.names)
|
||||
elif isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
modules.add(alias.name.split(".")[0])
|
||||
names.add(alias.name)
|
||||
return names, modules
|
||||
|
||||
|
||||
def test_page_modules_are_discovered():
|
||||
"""Guard the guard: the rules below are meaningless if nothing is scanned."""
|
||||
discovered = {path.stem for path in _page_paths()}
|
||||
assert discovered >= {"documents_page", "jobs_page", "people_page", "sources_page"}
|
||||
|
||||
|
||||
def test_no_page_imports_persistence_or_process_globals():
|
||||
"""HIGH-07: pages orchestrate services; they do not own sessions or settings."""
|
||||
violations: dict[str, list[str]] = {}
|
||||
for path in _page_paths():
|
||||
names, modules = _imports(ast.parse(path.read_text(encoding="utf-8")))
|
||||
found = sorted((names & FORBIDDEN_PAGE_IMPORTS) | (modules & FORBIDDEN_PAGE_MODULES))
|
||||
if found:
|
||||
violations[path.stem] = found
|
||||
assert violations == {}
|
||||
|
||||
|
||||
def test_no_component_resolves_request_or_application_state():
|
||||
"""`ui.instructions.md:31`: components render, they do not resolve app state."""
|
||||
violations: dict[str, list[str]] = {}
|
||||
for path in _component_paths():
|
||||
names, modules = _imports(ast.parse(path.read_text(encoding="utf-8")))
|
||||
found = sorted((names & FORBIDDEN_PAGE_IMPORTS) | (modules & (FORBIDDEN_PAGE_MODULES | {"fastapi"})))
|
||||
if found:
|
||||
violations[str(path.relative_to(COMPONENTS_DIR))] = found
|
||||
assert violations == {}
|
||||
|
||||
|
||||
# `build_table` owns the interactive table styling; `print_preview_page` owns the
|
||||
# print-only table, which must never paginate or expose a search box.
|
||||
TABLE_OWNERS = frozenset({"components/table/common.py", "pages/print_preview_page.py"})
|
||||
|
||||
|
||||
def _calls_ui_table(tree: ast.Module) -> bool:
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
func = node.func
|
||||
if (
|
||||
isinstance(func, ast.Attribute)
|
||||
and func.attr == "table"
|
||||
and isinstance(func.value, ast.Name)
|
||||
and func.value.id == "ui"
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def test_only_the_designated_owners_construct_a_raw_table():
|
||||
"""Review section 4: table styling lives in one place, not in every page."""
|
||||
offenders = sorted(
|
||||
str(path.relative_to(UI_DIR)).replace("\\", "/")
|
||||
for path in UI_DIR.rglob("*.py")
|
||||
if _calls_ui_table(ast.parse(path.read_text(encoding="utf-8")))
|
||||
)
|
||||
assert set(offenders) == TABLE_OWNERS
|
||||
@@ -77,6 +77,5 @@ def test_theme_defines_shared_semantic_surfaces():
|
||||
"ui-chip-primary",
|
||||
"ui-badge-secondary",
|
||||
"ui-status",
|
||||
"document-panzoom-host",
|
||||
):
|
||||
assert f".{class_name}" in theme_css
|
||||
|
||||
@@ -10,6 +10,7 @@ from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import JsonValue
|
||||
|
||||
from transcription.benchmarking import EditorialAssessment
|
||||
from transcription.benchmarking import score_transcription
|
||||
@@ -33,6 +34,18 @@ from transcription.services.sources import TranscriptionError
|
||||
from transcription.services.sources import transcribe_document_image
|
||||
|
||||
|
||||
def _json_object(value: JsonValue) -> dict[str, JsonValue]:
|
||||
"""Narrow a JSON export member to an object, asserting the export shape."""
|
||||
assert isinstance(value, dict)
|
||||
return value
|
||||
|
||||
|
||||
def _json_array(value: JsonValue) -> list[JsonValue]:
|
||||
"""Narrow a JSON export member to an array, asserting the export shape."""
|
||||
assert isinstance(value, list)
|
||||
return value
|
||||
|
||||
|
||||
class _ChunkedAsyncStream(httpx.AsyncByteStream):
|
||||
def __init__(self, chunks: list[bytes]):
|
||||
self._chunks = chunks
|
||||
@@ -261,9 +274,9 @@ async def test_attempts_are_append_only_and_exported_with_integrity(default_sess
|
||||
)
|
||||
)
|
||||
export = await sources.build_evidence_export(source_id=source.id)
|
||||
assert export["source"]["digest_sha256"] == "a" * 64
|
||||
assert [item["attempt_number"] for item in export["attempts"]] == [1, 2]
|
||||
assert export["artifacts"][0]["id"] == str(artifact.id)
|
||||
assert _json_object(export["source"])["digest_sha256"] == "a" * 64
|
||||
assert [_json_object(item)["attempt_number"] for item in _json_array(export["attempts"])] == [1, 2]
|
||||
assert _json_object(_json_array(export["artifacts"])[0])["id"] == str(artifact.id)
|
||||
assert "file_path" not in json.dumps(export)
|
||||
with pytest.raises(JobDeleteBlockedError):
|
||||
await jobs.delete_job_with_guardrails(job_id=job.id)
|
||||
@@ -275,7 +288,7 @@ async def test_attempts_are_append_only_and_exported_with_integrity(default_sess
|
||||
assert detail.processing_artifacts == []
|
||||
latest_attempt = await sources.read_latest_execution_attempt(job_source_id=latest_job_source.id)
|
||||
assert latest_attempt is not None
|
||||
assert latest_attempt.attempt_number == 2
|
||||
assert latest_attempt.attempt.attempt_number == 2
|
||||
|
||||
|
||||
def test_benchmark_scoring_preserves_literal_differences():
|
||||
|
||||
+67
-5
@@ -1,8 +1,11 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
from transcription.services import ServiceBundle
|
||||
from transcription.services.sources import SourceService
|
||||
from transcription.worker import process_next_queued_job
|
||||
from transcription.worker import run_worker_loop
|
||||
|
||||
@@ -12,9 +15,9 @@ async def test_run_worker_loop_survives_process_next_exception(monkeypatch, capl
|
||||
calls = 0
|
||||
stop_event = asyncio.Event()
|
||||
|
||||
async def _fake_process_next_queued_job(*, session=None, session_factory=None):
|
||||
async def _fake_process_next_queued_job(*, session=None, session_factory=None, services=None):
|
||||
nonlocal calls
|
||||
_ = (session, session_factory)
|
||||
_ = (session, session_factory, services)
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
raise RuntimeError("boom")
|
||||
@@ -31,7 +34,10 @@ async def test_run_worker_loop_survives_process_next_exception(monkeypatch, capl
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_next_closes_initialized_provider(monkeypatch):
|
||||
async def test_run_worker_loop_reuses_one_bundle_across_jobs(monkeypatch):
|
||||
"""HIGH-02: the provider client is built once per loop, not once per job."""
|
||||
stop_event = asyncio.Event()
|
||||
seen: list[object] = []
|
||||
closed = False
|
||||
|
||||
class _Sources:
|
||||
@@ -39,9 +45,43 @@ async def test_process_next_closes_initialized_provider(monkeypatch):
|
||||
nonlocal closed
|
||||
closed = True
|
||||
|
||||
services = type("_Services", (), {"sources": _Sources()})()
|
||||
bundle = ServiceBundle(sources=cast("SourceService", _Sources()))
|
||||
monkeypatch.setattr(
|
||||
"transcription.worker.ServiceBundle.from_session_factory",
|
||||
classmethod(lambda _cls, _factory=None, **_kwargs: bundle),
|
||||
)
|
||||
|
||||
monkeypatch.setattr("transcription.worker.ServiceBundle", lambda: services)
|
||||
async def _fake_process_next_queued_job(*, session=None, session_factory=None, services=None):
|
||||
_ = (session, session_factory)
|
||||
seen.append(services)
|
||||
if len(seen) >= 3:
|
||||
stop_event.set()
|
||||
return False
|
||||
return True
|
||||
|
||||
monkeypatch.setattr("transcription.worker.process_next_queued_job", _fake_process_next_queued_job)
|
||||
|
||||
await run_worker_loop(stop_event=stop_event, poll_interval_seconds=0)
|
||||
|
||||
assert len(seen) == 3
|
||||
assert all(item is bundle for item in seen)
|
||||
assert closed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_next_closes_provider_for_the_bundle_it_owns(monkeypatch):
|
||||
closed = False
|
||||
|
||||
class _Sources:
|
||||
async def aclose(self):
|
||||
nonlocal closed
|
||||
closed = True
|
||||
|
||||
bundle = ServiceBundle(sources=cast("SourceService", _Sources()))
|
||||
monkeypatch.setattr(
|
||||
"transcription.worker.ServiceBundle.from_session_factory",
|
||||
classmethod(lambda _cls, _factory=None, **_kwargs: bundle),
|
||||
)
|
||||
|
||||
async def _no_job(*, services, session):
|
||||
_ = (services, session)
|
||||
@@ -51,3 +91,25 @@ async def test_process_next_closes_initialized_provider(monkeypatch):
|
||||
|
||||
assert await process_next_queued_job() is False
|
||||
assert closed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_next_leaves_a_caller_owned_bundle_open(monkeypatch):
|
||||
"""A bundle passed in belongs to the caller and must outlive one job."""
|
||||
closed = False
|
||||
|
||||
class _Sources:
|
||||
async def aclose(self):
|
||||
nonlocal closed
|
||||
closed = True
|
||||
|
||||
bundle = ServiceBundle(sources=cast("SourceService", _Sources()))
|
||||
|
||||
async def _no_job(*, services, session):
|
||||
_ = (services, session)
|
||||
return False
|
||||
|
||||
monkeypatch.setattr("transcription.worker.process_next_queued_job_workflow", _no_job)
|
||||
|
||||
assert await process_next_queued_job(services=bundle) is False
|
||||
assert closed is False
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import Awaitable
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Generator
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -32,12 +32,13 @@ from transcription.db.models import Source
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def app_client(tmp_path_factory: pytest.TempPathFactory) -> AsyncGenerator[tuple[FastAPI, TestClient]]:
|
||||
def app_client(tmp_path_factory: pytest.TempPathFactory) -> Generator[tuple[FastAPI, TestClient]]:
|
||||
"""Provide a real application and test client backed by in-memory SQLite."""
|
||||
tmp_path = tmp_path_factory.mktemp("ui")
|
||||
database = SqliteSettings(path=str(tmp_path / "ui-tests.db"))
|
||||
settings = Settings(
|
||||
openrouter_api_key="test-key",
|
||||
database=SqliteSettings(path=str(tmp_path / "ui-tests.db")),
|
||||
database=database,
|
||||
environment="test",
|
||||
bootstrap_schema_on_startup=True,
|
||||
upload_dir=tmp_path / "uploads",
|
||||
@@ -48,7 +49,7 @@ def app_client(tmp_path_factory: pytest.TempPathFactory) -> AsyncGenerator[tuple
|
||||
|
||||
with TestClient(app) as client:
|
||||
runtime_path = Path(str(app.state.runtime.engine.url.database)).resolve()
|
||||
expected_path = Path(settings.database.path).resolve()
|
||||
expected_path = Path(database.path).resolve()
|
||||
if runtime_path != expected_path:
|
||||
raise RuntimeError(
|
||||
"Refusing to initialize destructive UI fixtures against "
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Homepage storage resolves its root from settings rather than from `__file__`.
|
||||
|
||||
The previous module derived its directory from ``Path(__file__).parents[3]``,
|
||||
which could not be configured and resolved into the installed package directory
|
||||
outside a source checkout.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from transcription.config import Settings
|
||||
from transcription.ui.homepage_store import homepage_dir
|
||||
from transcription.ui.homepage_store import latest_homepage_image
|
||||
from transcription.ui.homepage_store import list_homepage_images
|
||||
from transcription.ui.homepage_store import read_homepage_markdown
|
||||
from transcription.ui.homepage_store import save_homepage_markdown
|
||||
from transcription.ui.homepage_store import store_homepage_image
|
||||
|
||||
PNG_BYTES = bytes.fromhex(
|
||||
"89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000a49444154789c6360000002000100"
|
||||
"05fe02fea7b1b8000000004945"
|
||||
) + b"NDAE\xae\x42\x60\x82"
|
||||
|
||||
|
||||
def _settings(tmp_path) -> Settings:
|
||||
return Settings(openrouter_api_key="test-key-abc123", homepage_dir=tmp_path / "homepage")
|
||||
|
||||
|
||||
def test_homepage_dir_follows_the_configured_setting(tmp_path):
|
||||
settings = _settings(tmp_path)
|
||||
|
||||
assert homepage_dir(settings) == tmp_path / "homepage"
|
||||
|
||||
|
||||
def test_markdown_round_trips_through_the_configured_directory(tmp_path):
|
||||
settings = _settings(tmp_path)
|
||||
|
||||
assert read_homepage_markdown(settings) == ""
|
||||
|
||||
save_homepage_markdown("# Archive", settings)
|
||||
|
||||
assert (tmp_path / "homepage" / "homepage.md").read_text(encoding="utf-8") == "# Archive"
|
||||
assert read_homepage_markdown(settings) == "# Archive"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_images_are_stored_and_listed_from_the_configured_directory(tmp_path):
|
||||
settings = _settings(tmp_path)
|
||||
|
||||
assert list_homepage_images(settings) == []
|
||||
assert latest_homepage_image(settings) is None
|
||||
|
||||
stored = await store_homepage_image(filename="banner.png", file_bytes=PNG_BYTES, settings=settings)
|
||||
|
||||
assert stored.parent == tmp_path / "homepage"
|
||||
assert list_homepage_images(settings) == [stored]
|
||||
assert latest_homepage_image(settings) == stored
|
||||
|
||||
|
||||
def test_two_configurations_do_not_share_storage(tmp_path):
|
||||
first = Settings(openrouter_api_key="test-key-abc123", homepage_dir=tmp_path / "a")
|
||||
second = Settings(openrouter_api_key="test-key-abc123", homepage_dir=tmp_path / "b")
|
||||
|
||||
save_homepage_markdown("first", first)
|
||||
|
||||
assert read_homepage_markdown(second) == ""
|
||||
@@ -1,11 +1,13 @@
|
||||
"""Tests for the sources page routes and Source model properties."""
|
||||
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlmodel import select
|
||||
|
||||
from transcription.db import session_scope
|
||||
from transcription.db.loading import selectinload
|
||||
from transcription.db.models import Document
|
||||
from transcription.db.models import Job
|
||||
from transcription.db.models import JobSourceStatus
|
||||
@@ -23,6 +25,7 @@ class TestSourceModelProperties:
|
||||
@pytest.mark.asyncio
|
||||
async def test_source_properties_with_no_job_sources(self):
|
||||
source = Source(
|
||||
document_id=uuid4(),
|
||||
page_number=1,
|
||||
upload_name="page_one.png",
|
||||
filename="stored_page_one.png",
|
||||
@@ -48,7 +51,16 @@ class TestSourceModelProperties:
|
||||
async with session_scope() as session:
|
||||
job = await session.get(Job, job_id)
|
||||
assert job is not None
|
||||
source = (await session.exec(select(Source).where(Source.document_id == job.document_id))).first()
|
||||
source = (
|
||||
await session.exec(
|
||||
select(Source)
|
||||
.options(
|
||||
selectinload(Source.document),
|
||||
selectinload(Source.job_sources),
|
||||
)
|
||||
.where(Source.document_id == job.document_id)
|
||||
)
|
||||
).first()
|
||||
assert source is not None
|
||||
|
||||
# Validate computed properties
|
||||
|
||||
@@ -41,4 +41,3 @@ class TestPageRendering:
|
||||
assert response.status_code == 200
|
||||
assert "Edit Home Page" in response.text
|
||||
assert "Homepage markdown" in response.text
|
||||
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
"""One-time migration of a V4.5 database into the re-leveled V4.6 schema.
|
||||
|
||||
V4.6 re-levels the schema from the current SQLModel metadata rather than
|
||||
running a chain of hand-rolled upgrade functions. The column sets are
|
||||
unchanged; what changed is index coverage ([HIGH-04]), the ``use_alter``
|
||||
break in the ``source``/``execution_attempt`` foreign key cycle, and the
|
||||
relationship loading strategy ([CRIT-02]). This script therefore performs a
|
||||
faithful, foreign-key-ordered row copy.
|
||||
|
||||
Design notes:
|
||||
|
||||
- The backup is read with plain ``sqlite3`` rather than through the ORM. The
|
||||
V4.5 file is not guaranteed to satisfy the V4.6 mappers, and reading raw
|
||||
rows means no relationship is ever traversed, so ``lazy="raise"`` cannot
|
||||
bite.
|
||||
- The target is written through SQLAlchemy Core against the live metadata, so
|
||||
the same script works against PostgreSQL when that cutover happens.
|
||||
- Identity is preserved exactly: UUIDs, digests, timestamps, attempt numbers,
|
||||
and ``preferred_execution_attempt_id`` selections carry across unchanged.
|
||||
No evidence payload is reinterpreted, normalized, or regenerated.
|
||||
- No on-disk Source file, portrait, or artifact file is read for writing or
|
||||
modified. ``--verify-artifacts`` reads artifact files, but only to hash
|
||||
them.
|
||||
- The script is idempotent: a row whose primary key already exists in the
|
||||
target is skipped, never updated. It is never invoked from application
|
||||
startup and never runs in the test suite.
|
||||
|
||||
Usage::
|
||||
|
||||
python tools/migrate_v45_to_v46.py --dry-run
|
||||
python tools/migrate_v45_to_v46.py --verify-artifacts
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Sequence
|
||||
from datetime import date
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import Column
|
||||
from sqlalchemy import Table
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import insert
|
||||
from sqlalchemy import inspect as sqlalchemy_inspect
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import update
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlmodel import SQLModel
|
||||
|
||||
from transcription.config import Settings
|
||||
from transcription.config import get_settings
|
||||
from transcription.db import models as _models # noqa: F401 (registers every table)
|
||||
from transcription.db.engine import get_database_url
|
||||
|
||||
DEFAULT_BACKUP = Path("data/transcription.db.pre-v46.bak")
|
||||
|
||||
#: ``source.preferred_execution_attempt_id`` points at ``execution_attempt``,
|
||||
#: which points back at ``source``. The cycle is broken with ``use_alter`` in
|
||||
#: the metadata, so ``source`` rows are inserted with the column cleared and
|
||||
#: the selections are replayed once ``execution_attempt`` is populated.
|
||||
DEFERRED_TABLE = "source"
|
||||
DEFERRED_COLUMN = "preferred_execution_attempt_id"
|
||||
|
||||
#: Row counts the V4.5 backup is expected to carry, used as a pre-flight guard
|
||||
#: so the script cannot silently run against the wrong file.
|
||||
EXPECTED_SOURCE_COUNTS = {
|
||||
"document": 8,
|
||||
"document_person": 11,
|
||||
"document_type": 7,
|
||||
"execution_attempt": 80,
|
||||
"job": 11,
|
||||
"job_source": 79,
|
||||
"person": 5,
|
||||
"person_role": 3,
|
||||
"processing_artifact": 2,
|
||||
"source": 76,
|
||||
}
|
||||
|
||||
|
||||
def _coerce(column: Column[Any], value: object) -> object:
|
||||
"""Convert a raw SQLite value into what the target column's type binds.
|
||||
|
||||
SQLite hands back strings and integers; the V4.6 columns bind ``UUID``,
|
||||
``datetime``, ``date``, ``bool``, enum members, and decoded JSON. The
|
||||
conversion is lossless in both directions.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
match type(column.type).__name__:
|
||||
case "Uuid":
|
||||
return value if isinstance(value, UUID) else UUID(str(value))
|
||||
case "DateTime":
|
||||
return value if isinstance(value, datetime) else datetime.fromisoformat(str(value))
|
||||
case "Date":
|
||||
return value if isinstance(value, date) else date.fromisoformat(str(value))
|
||||
case "Boolean":
|
||||
return bool(value)
|
||||
case "JSONBCompat":
|
||||
if isinstance(value, str | bytes | bytearray):
|
||||
return json.loads(value)
|
||||
return value
|
||||
case "Enum":
|
||||
enum_class = getattr(column.type, "enum_class", None)
|
||||
if enum_class is None:
|
||||
return value
|
||||
# The same JobSourceStatus enum is persisted by value on
|
||||
# job_source.status and by name on execution_attempt.status,
|
||||
# because only the former declares values_callable. Accept either
|
||||
# spelling so the copy round-trips both columns faithfully.
|
||||
try:
|
||||
return enum_class(value)
|
||||
except ValueError:
|
||||
return enum_class[str(value)]
|
||||
case _:
|
||||
return value
|
||||
|
||||
|
||||
def _read_table(backup: sqlite3.Connection, table: Table) -> list[dict[str, object]]:
|
||||
"""Read every row of ``table`` from the backup, coerced for the target."""
|
||||
names = [column.name for column in table.columns]
|
||||
quoted = ", ".join(f'"{name}"' for name in names)
|
||||
rows: list[dict[str, object]] = []
|
||||
for raw in backup.execute(f'select {quoted} from "{table.name}"'):
|
||||
rows.append({name: _coerce(table.columns[name], raw[index]) for index, name in enumerate(names)})
|
||||
return rows
|
||||
|
||||
|
||||
def _primary_key(table: Table) -> Column[Any]:
|
||||
columns = list(table.primary_key.columns)
|
||||
if len(columns) != 1:
|
||||
message = f"{table.name} does not have a single-column primary key"
|
||||
raise RuntimeError(message)
|
||||
return columns[0]
|
||||
|
||||
|
||||
def _existing_keys(connection: Connection, table: Table) -> set[object]:
|
||||
key = _primary_key(table)
|
||||
return set(connection.execute(select(key)).scalars().all())
|
||||
|
||||
|
||||
def _chunked(rows: Sequence[dict[str, object]], size: int = 200) -> Iterator[Sequence[dict[str, object]]]:
|
||||
for start in range(0, len(rows), size):
|
||||
yield rows[start : start + size]
|
||||
|
||||
|
||||
def _verify_artifacts(settings: Settings) -> int:
|
||||
"""Re-hash every migrated artifact through the service's own verifier."""
|
||||
from transcription.db.models import ProcessingArtifact
|
||||
from transcription.services.sources import SourceService
|
||||
|
||||
engine = create_engine(_sync_url(settings))
|
||||
with engine.connect() as connection:
|
||||
rows = connection.execute(select(SQLModel.metadata.tables["processing_artifact"])).mappings().all()
|
||||
engine.dispose()
|
||||
|
||||
service = SourceService(settings=settings)
|
||||
artifacts = [ProcessingArtifact(**dict(row)) for row in rows]
|
||||
# Reuses the application's own integrity check so the migration cannot
|
||||
# disagree with what the running app considers a valid artifact.
|
||||
service._verify_artifacts_integrity(artifacts)
|
||||
return len(artifacts)
|
||||
|
||||
|
||||
def _sync_url(settings: Settings) -> str:
|
||||
"""Return the target database URL with any async driver stripped."""
|
||||
url = get_database_url(settings)
|
||||
return url.replace("+aiosqlite", "").replace("+asyncpg", "").replace("+psycopg", "")
|
||||
|
||||
|
||||
def _preflight(backup: sqlite3.Connection, *, strict: bool) -> None:
|
||||
actual = {
|
||||
name: backup.execute(f'select count(*) from "{name}"').fetchone()[0]
|
||||
for name in EXPECTED_SOURCE_COUNTS
|
||||
}
|
||||
mismatched = {
|
||||
name: (count, EXPECTED_SOURCE_COUNTS[name])
|
||||
for name, count in actual.items()
|
||||
if count != EXPECTED_SOURCE_COUNTS[name]
|
||||
}
|
||||
if not mismatched:
|
||||
return
|
||||
detail = ", ".join(f"{name}: found {found}, expected {want}" for name, (found, want) in sorted(mismatched.items()))
|
||||
message = f"Backup row counts do not match the recorded V4.5 snapshot ({detail})"
|
||||
if strict:
|
||||
raise RuntimeError(message)
|
||||
print(f"WARNING: {message}", file=sys.stderr)
|
||||
|
||||
|
||||
def _copy_tables(
|
||||
connection: Connection,
|
||||
payload: dict[str, list[dict[str, object]]],
|
||||
*,
|
||||
dry_run: bool,
|
||||
) -> tuple[int, dict[object, object]]:
|
||||
"""Insert every missing row, deferring the cyclic foreign key column."""
|
||||
deferred: dict[object, object] = {}
|
||||
inserted_total = 0
|
||||
|
||||
for table in SQLModel.metadata.sorted_tables:
|
||||
rows = payload[table.name]
|
||||
existing = set() if dry_run else _existing_keys(connection, table)
|
||||
key_name = _primary_key(table).name
|
||||
|
||||
pending = [row for row in rows if row[key_name] not in existing]
|
||||
|
||||
if table.name == DEFERRED_TABLE:
|
||||
for row in pending:
|
||||
selection = row[DEFERRED_COLUMN]
|
||||
if selection is not None:
|
||||
deferred[row[key_name]] = selection
|
||||
row[DEFERRED_COLUMN] = None
|
||||
|
||||
if pending and not dry_run:
|
||||
for chunk in _chunked(pending):
|
||||
connection.execute(insert(table), list(chunk))
|
||||
|
||||
inserted_total += len(pending)
|
||||
print(f" {table.name:24} insert={len(pending):<5} skip={len(rows) - len(pending)}")
|
||||
|
||||
return inserted_total, deferred
|
||||
|
||||
|
||||
def _replay_deferred(connection: Connection, deferred: dict[object, object], *, dry_run: bool) -> None:
|
||||
"""Restore the preferred-attempt selections held back by the FK cycle."""
|
||||
if not deferred:
|
||||
return
|
||||
print(f" replaying {len(deferred)} deferred {DEFERRED_TABLE}.{DEFERRED_COLUMN} selection(s)")
|
||||
if dry_run:
|
||||
return
|
||||
source = SQLModel.metadata.tables[DEFERRED_TABLE]
|
||||
key = _primary_key(source)
|
||||
for source_id, attempt_id in deferred.items():
|
||||
connection.execute(update(source).where(key == source_id).values({DEFERRED_COLUMN: attempt_id}))
|
||||
|
||||
|
||||
def _report_counts(connection: Connection) -> None:
|
||||
print("\nPost-migration row counts:")
|
||||
for table in SQLModel.metadata.sorted_tables:
|
||||
actual = len(connection.execute(select(_primary_key(table))).all())
|
||||
expected = EXPECTED_SOURCE_COUNTS.get(table.name)
|
||||
flag = "" if expected is None or actual == expected else f" <-- expected {expected}"
|
||||
print(f" {table.name:24} {actual}{flag}")
|
||||
|
||||
|
||||
def _load_payload(backup_path: Path, *, strict_counts: bool) -> dict[str, list[dict[str, object]]]:
|
||||
if not backup_path.is_file():
|
||||
message = f"Backup database not found: {backup_path}"
|
||||
raise FileNotFoundError(message)
|
||||
backup = sqlite3.connect(f"file:{backup_path}?mode=ro", uri=True)
|
||||
try:
|
||||
_preflight(backup, strict=strict_counts)
|
||||
return {table.name: _read_table(backup, table) for table in SQLModel.metadata.sorted_tables}
|
||||
finally:
|
||||
backup.close()
|
||||
|
||||
|
||||
def migrate(*, backup_path: Path, settings: Settings, dry_run: bool, strict_counts: bool) -> int:
|
||||
"""Copy every row from the V4.5 backup into the re-leveled schema."""
|
||||
payload = _load_payload(backup_path, strict_counts=strict_counts)
|
||||
|
||||
engine = create_engine(_sync_url(settings))
|
||||
try:
|
||||
if not sqlalchemy_inspect(engine).has_table("document"):
|
||||
print("Target schema is empty; creating it from the current metadata.")
|
||||
if not dry_run:
|
||||
SQLModel.metadata.create_all(engine)
|
||||
|
||||
with engine.begin() as connection:
|
||||
inserted_total, deferred = _copy_tables(connection, payload, dry_run=dry_run)
|
||||
_replay_deferred(connection, deferred, dry_run=dry_run)
|
||||
|
||||
if dry_run:
|
||||
print("\nDry run: no rows were written.")
|
||||
return inserted_total
|
||||
|
||||
with engine.connect() as connection:
|
||||
_report_counts(connection)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
return inserted_total
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--backup", type=Path, default=DEFAULT_BACKUP, help="V4.5 database to read from")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Report what would be copied without writing")
|
||||
parser.add_argument(
|
||||
"--allow-count-mismatch",
|
||||
action="store_true",
|
||||
help="Warn instead of aborting when the backup row counts differ from the recorded snapshot",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verify-artifacts",
|
||||
action="store_true",
|
||||
help="Re-hash every migrated processing artifact after the copy",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
settings = get_settings()
|
||||
print(f"Source: {args.backup}")
|
||||
print(f"Target: {_sync_url(settings)}\n")
|
||||
|
||||
inserted = migrate(
|
||||
backup_path=args.backup,
|
||||
settings=settings,
|
||||
dry_run=args.dry_run,
|
||||
strict_counts=not args.allow_count_mismatch,
|
||||
)
|
||||
|
||||
if args.verify_artifacts and not args.dry_run:
|
||||
verified = _verify_artifacts(settings)
|
||||
print(f"\nArtifact integrity verified for {verified} artifact(s).")
|
||||
|
||||
print(f"\nDone. {inserted} row(s) inserted.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -7,7 +7,6 @@ import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from ctypes import wintypes
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
@@ -20,17 +19,16 @@ def show_phase(title: str) -> None:
|
||||
print(f"========== {title} ==========")
|
||||
|
||||
|
||||
def test_file_unlocked(path: Path) -> bool:
|
||||
if sys.platform == "win32":
|
||||
# `ctypes.wintypes` raises on import off Windows, and `fcntl` does not exist
|
||||
# on Windows, so the two implementations are selected at module level where a
|
||||
# type checker can narrow `sys.platform` and analyze only the live branch.
|
||||
from ctypes import wintypes
|
||||
|
||||
def test_file_unlocked(path: Path) -> bool:
|
||||
if not path.exists():
|
||||
return True
|
||||
|
||||
if os.name == "nt":
|
||||
return _test_file_unlocked_windows(path)
|
||||
|
||||
return _test_file_unlocked_posix(path)
|
||||
|
||||
|
||||
def _test_file_unlocked_windows(path: Path) -> bool:
|
||||
generic_read = 0x80000000
|
||||
generic_write = 0x40000000
|
||||
open_existing = 3
|
||||
@@ -66,10 +64,13 @@ def _test_file_unlocked_windows(path: Path) -> bool:
|
||||
kernel32.CloseHandle(handle)
|
||||
return True
|
||||
|
||||
|
||||
def _test_file_unlocked_posix(path: Path) -> bool:
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
def test_file_unlocked(path: Path) -> bool:
|
||||
if not path.exists():
|
||||
return True
|
||||
|
||||
fd = os.open(path, os.O_RDWR)
|
||||
try:
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user