generated from john/python-template
breaking up pages
This commit is contained in:
@@ -7,24 +7,27 @@ from threading import Event
|
|||||||
from threading import Thread
|
from threading import Thread
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||||
|
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||||
|
|
||||||
from .api.errors import register_error_handlers
|
from .api.errors import register_error_handlers
|
||||||
from .api.health import router as health_router
|
from .api.health import router as health_router
|
||||||
|
from .config import configure_logging
|
||||||
from .config import get_settings
|
from .config import get_settings
|
||||||
from .config import setup_logging
|
|
||||||
from .db import cleanup_database
|
from .db import cleanup_database
|
||||||
from .db import create_all
|
from .db import create_all
|
||||||
from .db import get_engine
|
from .db import initialize_database_runtime
|
||||||
from .ui import register_pages
|
from .ui import register_pages
|
||||||
from .worker import run_worker_loop
|
from .worker import run_worker_loop
|
||||||
|
|
||||||
|
|
||||||
def _start_worker(app: FastAPI) -> None:
|
def _start_worker(app: FastAPI) -> None:
|
||||||
|
session_factory: async_sessionmaker[AsyncSession] = app.state.db_session_factory
|
||||||
stop_event = Event()
|
stop_event = Event()
|
||||||
worker_thread = Thread(
|
worker_thread = Thread(
|
||||||
target=run_worker_loop,
|
target=run_worker_loop,
|
||||||
kwargs={
|
kwargs={
|
||||||
"engine": app.state.db_conn,
|
"session_factory": session_factory,
|
||||||
"stop_event": stop_event,
|
"stop_event": stop_event,
|
||||||
"poll_interval_seconds": 1.0,
|
"poll_interval_seconds": 1.0,
|
||||||
},
|
},
|
||||||
@@ -47,15 +50,16 @@ def _stop_worker(app: FastAPI) -> None:
|
|||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def _lifespan(app: FastAPI):
|
async def _lifespan(app: FastAPI):
|
||||||
setup_logging()
|
configure_logging()
|
||||||
|
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
app.state.settings = settings
|
app.state.settings = settings
|
||||||
engine = get_engine()
|
runtime = initialize_database_runtime(settings=settings)
|
||||||
app.state.db_conn = engine
|
app.state.db_engine = runtime.engine
|
||||||
|
app.state.db_session_factory = runtime.session_factory
|
||||||
|
|
||||||
if settings.should_bootstrap_schema:
|
if settings.should_bootstrap_schema:
|
||||||
create_all(engine=engine)
|
await create_all(engine=runtime.engine)
|
||||||
|
|
||||||
settings.upload_dir.mkdir(parents=True, exist_ok=True)
|
settings.upload_dir.mkdir(parents=True, exist_ok=True)
|
||||||
settings.prompt_dir.mkdir(parents=True, exist_ok=True)
|
settings.prompt_dir.mkdir(parents=True, exist_ok=True)
|
||||||
@@ -65,7 +69,7 @@ async def _lifespan(app: FastAPI):
|
|||||||
yield
|
yield
|
||||||
finally:
|
finally:
|
||||||
_stop_worker(app)
|
_stop_worker(app)
|
||||||
cleanup_database()
|
await cleanup_database()
|
||||||
|
|
||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
|
|||||||
@@ -99,7 +99,7 @@ LOGGING_CONFIG: dict[str, object] = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def setup_logging() -> None:
|
def configure_logging() -> None:
|
||||||
"""Configure root logging once at startup."""
|
"""Configure root logging once at startup."""
|
||||||
logging.config.dictConfig(LOGGING_CONFIG)
|
logging.config.dictConfig(LOGGING_CONFIG)
|
||||||
logger.debug("Logging configured")
|
logger.debug("Logging configured")
|
||||||
|
|||||||
+90
-30
@@ -4,17 +4,21 @@ V1 moves database resource ownership to explicit runtime initialization so
|
|||||||
startup/shutdown behavior is predictable and lifespan-managed.
|
startup/shutdown behavior is predictable and lifespan-managed.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
import contextlib
|
import contextlib
|
||||||
import logging
|
import logging
|
||||||
from collections.abc import Generator
|
from collections.abc import AsyncGenerator
|
||||||
from functools import cache
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from sqlalchemy import inspect
|
from sqlalchemy import inspect
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
from sqlalchemy.engine import Engine
|
from sqlalchemy.engine import Connection
|
||||||
from sqlmodel import Session
|
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||||
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||||
|
from sqlalchemy.ext.asyncio import create_async_engine
|
||||||
from sqlmodel import SQLModel
|
from sqlmodel import SQLModel
|
||||||
from sqlmodel import create_engine
|
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||||
|
|
||||||
from .config import Settings
|
from .config import Settings
|
||||||
from .config import get_settings
|
from .config import get_settings
|
||||||
@@ -22,66 +26,122 @@ from .config import get_settings
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _build_engine(settings: Settings) -> Engine:
|
@dataclass(frozen=True)
|
||||||
|
class DatabaseRuntime:
|
||||||
|
"""Database runtime resources owned by app lifespan."""
|
||||||
|
|
||||||
|
engine: AsyncEngine
|
||||||
|
session_factory: async_sessionmaker[AsyncSession]
|
||||||
|
|
||||||
|
|
||||||
|
_runtime: DatabaseRuntime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _to_async_database_url(database_url: str) -> str:
|
||||||
|
"""Normalize configured database URL to an async SQLAlchemy driver URL."""
|
||||||
|
if database_url.startswith("sqlite://") and not database_url.startswith("sqlite+aiosqlite://"):
|
||||||
|
return database_url.replace("sqlite://", "sqlite+aiosqlite://", 1)
|
||||||
|
if database_url.startswith("postgresql://") and not database_url.startswith("postgresql+asyncpg://"):
|
||||||
|
return database_url.replace("postgresql://", "postgresql+asyncpg://", 1)
|
||||||
|
return database_url
|
||||||
|
|
||||||
|
|
||||||
|
def _build_engine(settings: Settings) -> AsyncEngine:
|
||||||
|
database_url = _to_async_database_url(settings.database_url)
|
||||||
connect_args: dict[str, object] = {}
|
connect_args: dict[str, object] = {}
|
||||||
if settings.database_url.startswith("sqlite"):
|
if database_url.startswith("sqlite"):
|
||||||
connect_args["check_same_thread"] = False
|
connect_args["check_same_thread"] = False
|
||||||
return create_engine(
|
return create_async_engine(
|
||||||
url=settings.database_url,
|
url=database_url,
|
||||||
echo=False,
|
echo=False,
|
||||||
|
pool_pre_ping=True,
|
||||||
connect_args=connect_args,
|
connect_args=connect_args,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@cache
|
def initialize_database_runtime(*, settings: Settings | None = None) -> DatabaseRuntime:
|
||||||
def get_engine() -> Engine:
|
"""Initialize lifespan-owned async DB resources once per process."""
|
||||||
"""Return a new SQLAlchemy engine for the current settings."""
|
global _runtime
|
||||||
settings = get_settings()
|
if _runtime is not None:
|
||||||
logger.debug("Creating new SQLAlchemy engine for database_url=%s", settings.database_url)
|
return _runtime
|
||||||
return _build_engine(settings)
|
|
||||||
|
active_settings = settings or get_settings()
|
||||||
|
engine = _build_engine(active_settings)
|
||||||
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||||
|
_runtime = DatabaseRuntime(engine=engine, session_factory=session_factory)
|
||||||
|
logger.debug("Initialized async database runtime for database_url=%s", engine.url)
|
||||||
|
return _runtime
|
||||||
|
|
||||||
|
|
||||||
def cleanup_database() -> None:
|
def get_engine() -> AsyncEngine:
|
||||||
|
"""Return the current async SQLAlchemy engine."""
|
||||||
|
runtime = _runtime or initialize_database_runtime()
|
||||||
|
return runtime.engine
|
||||||
|
|
||||||
|
|
||||||
|
def get_session_factory() -> async_sessionmaker[AsyncSession]:
|
||||||
|
"""Return the shared async session factory."""
|
||||||
|
runtime = _runtime or initialize_database_runtime()
|
||||||
|
return runtime.session_factory
|
||||||
|
|
||||||
|
|
||||||
|
async def cleanup_database() -> None:
|
||||||
"""Cleanup database runtime resources."""
|
"""Cleanup database runtime resources."""
|
||||||
engine = get_engine()
|
await dispose_database_runtime()
|
||||||
engine.dispose()
|
|
||||||
|
|
||||||
|
|
||||||
def create_all(*, engine: Engine | None = None) -> None:
|
async def dispose_database_runtime() -> None:
|
||||||
|
"""Dispose lifespan-owned async database resources."""
|
||||||
|
global _runtime
|
||||||
|
if _runtime is None:
|
||||||
|
return
|
||||||
|
await _runtime.engine.dispose()
|
||||||
|
_runtime = None
|
||||||
|
|
||||||
|
|
||||||
|
async def create_all(*, engine: AsyncEngine | None = None) -> None:
|
||||||
"""Create all tables on the selected engine."""
|
"""Create all tables on the selected engine."""
|
||||||
# Import models so SQLModel metadata is fully registered before bootstrap.
|
# Import models so SQLModel metadata is fully registered before bootstrap.
|
||||||
from transcription import models as _models # noqa: F401
|
from transcription import models as _models # noqa: F401
|
||||||
|
|
||||||
active_engine = engine or get_engine()
|
active_engine = engine or get_engine()
|
||||||
SQLModel.metadata.create_all(active_engine)
|
async with active_engine.begin() as connection:
|
||||||
_ensure_sqlite_compat_columns(active_engine)
|
await connection.run_sync(SQLModel.metadata.create_all)
|
||||||
|
await connection.run_sync(_ensure_sqlite_compat_columns)
|
||||||
logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url)
|
logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url)
|
||||||
|
|
||||||
|
|
||||||
def _ensure_sqlite_compat_columns(engine: Engine) -> None:
|
def _ensure_sqlite_compat_columns(connection: Connection) -> None:
|
||||||
"""Apply lightweight dev/test SQLite compatibility column patches.
|
"""Apply lightweight dev/test SQLite compatibility column patches.
|
||||||
|
|
||||||
This keeps local bootstrap resilient when models evolve but no full
|
This keeps local bootstrap resilient when models evolve but no full
|
||||||
migration tooling is in place yet.
|
migration tooling is in place yet.
|
||||||
"""
|
"""
|
||||||
if engine.url.get_backend_name() != "sqlite":
|
if connection.engine.url.get_backend_name() != "sqlite":
|
||||||
return
|
return
|
||||||
|
|
||||||
inspector = inspect(engine)
|
inspector = inspect(connection)
|
||||||
table_names = set(inspector.get_table_names())
|
table_names = set(inspector.get_table_names())
|
||||||
if "job" not in table_names:
|
if "job" not in table_names:
|
||||||
return
|
return
|
||||||
|
|
||||||
columns = {column["name"] for column in inspector.get_columns("job")}
|
columns = {column["name"] for column in inspector.get_columns("job")}
|
||||||
if "retry_count" not in columns:
|
if "retry_count" not in columns:
|
||||||
with engine.begin() as connection:
|
connection.execute(text("ALTER TABLE job ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0"))
|
||||||
connection.execute(text("ALTER TABLE job ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0"))
|
|
||||||
logger.warning("Applied SQLite compatibility schema patch table=job column=retry_count default=0")
|
logger.warning("Applied SQLite compatibility schema patch table=job column=retry_count default=0")
|
||||||
|
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.asynccontextmanager
|
||||||
def get_session(*, engine: Engine | None = None) -> Generator[Session]:
|
async def get_session(
|
||||||
|
*,
|
||||||
|
session_factory: async_sessionmaker[AsyncSession] | None = None,
|
||||||
|
) -> AsyncGenerator[AsyncSession]:
|
||||||
"""Yield a database session and ensure cleanup."""
|
"""Yield a database session and ensure cleanup."""
|
||||||
active_engine = engine or get_engine()
|
active_session_factory = session_factory or get_session_factory()
|
||||||
with Session(active_engine) as session:
|
async with active_session_factory() as session:
|
||||||
yield session
|
yield session
|
||||||
|
|
||||||
|
|
||||||
|
def should_bootstrap_schema(settings: Settings) -> bool:
|
||||||
|
"""Compatibility helper for explicit bootstrap checks."""
|
||||||
|
return settings.should_bootstrap_schema
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from pathlib import Path
|
|||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from sqlmodel import Session
|
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||||
|
|
||||||
from transcription.config import Settings
|
from transcription.config import Settings
|
||||||
from transcription.config import get_settings
|
from transcription.config import get_settings
|
||||||
@@ -38,11 +38,11 @@ class UploadJobResult:
|
|||||||
original_filename: str
|
original_filename: str
|
||||||
|
|
||||||
|
|
||||||
def create_upload_job(
|
async def create_upload_job(
|
||||||
*,
|
*,
|
||||||
filename: str,
|
filename: str,
|
||||||
file_bytes: bytes,
|
file_bytes: bytes,
|
||||||
session: Session | None = None,
|
session: AsyncSession | None = None,
|
||||||
settings: Settings | None = None,
|
settings: Settings | None = None,
|
||||||
) -> UploadJobResult:
|
) -> UploadJobResult:
|
||||||
"""Persist an uploaded file and create document/job records."""
|
"""Persist an uploaded file and create document/job records."""
|
||||||
@@ -66,10 +66,14 @@ def create_upload_job(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
if session is not None:
|
if session is not None:
|
||||||
document, job = _create_upload_records(session=session, original_filename=filename, stored_path=stored_path)
|
document, job = await _create_upload_records(
|
||||||
|
session=session,
|
||||||
|
original_filename=filename,
|
||||||
|
stored_path=stored_path,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
with get_session() as local_session:
|
async with get_session() as local_session:
|
||||||
document, job = _create_upload_records(
|
document, job = await _create_upload_records(
|
||||||
session=local_session,
|
session=local_session,
|
||||||
original_filename=filename,
|
original_filename=filename,
|
||||||
stored_path=stored_path,
|
stored_path=stored_path,
|
||||||
@@ -122,22 +126,27 @@ def _build_stored_filename(filename: str) -> str:
|
|||||||
return f"{uuid4()}_{safe_name}"
|
return f"{uuid4()}_{safe_name}"
|
||||||
|
|
||||||
|
|
||||||
def _create_upload_records(*, session: Session, original_filename: str, stored_path: Path) -> tuple[Document, Job]:
|
async def _create_upload_records(
|
||||||
|
*,
|
||||||
|
session: AsyncSession,
|
||||||
|
original_filename: str,
|
||||||
|
stored_path: Path,
|
||||||
|
) -> tuple[Document, Job]:
|
||||||
document = Document(
|
document = Document(
|
||||||
filename=Path(original_filename).name,
|
filename=Path(original_filename).name,
|
||||||
file_path=str(stored_path),
|
file_path=str(stored_path),
|
||||||
)
|
)
|
||||||
session.add(document)
|
session.add(document)
|
||||||
session.flush()
|
await session.flush()
|
||||||
|
|
||||||
job = Job(
|
job = Job(
|
||||||
document_id=document.id,
|
document_id=document.id,
|
||||||
status=JobStatus.QUEUED,
|
status=JobStatus.QUEUED,
|
||||||
)
|
)
|
||||||
session.add(job)
|
session.add(job)
|
||||||
session.commit()
|
await session.commit()
|
||||||
session.refresh(document)
|
await session.refresh(document)
|
||||||
session.refresh(job)
|
await session.refresh(job)
|
||||||
return document, job
|
return document, job
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -3,8 +3,8 @@
|
|||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from nicegui import ui
|
from nicegui import ui
|
||||||
|
|
||||||
from transcription.ui.jobs_page import register_page as register_jobs_page
|
from transcription.ui.pages.jobs_page import register_page as register_jobs_page
|
||||||
from transcription.ui.upload_page import register_page as register_upload_page
|
from transcription.ui.pages.upload_page import register_page as register_upload_page
|
||||||
|
|
||||||
|
|
||||||
def register_pages(app: FastAPI) -> None:
|
def register_pages(app: FastAPI) -> None:
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
"""Reusable job detail rendering helpers."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from nicegui import ui
|
||||||
|
|
||||||
|
from transcription.models import Document
|
||||||
|
from transcription.models import Job
|
||||||
|
from transcription.models import Transcript
|
||||||
|
|
||||||
|
|
||||||
|
def render_job_detail(*, job: Job, document: Document | None, transcript: Transcript | None) -> None:
|
||||||
|
"""Render all sections for the job detail page."""
|
||||||
|
ui.label(f"Job ID: {job.id}")
|
||||||
|
ui.label(f"Status: {job.status.value}")
|
||||||
|
ui.label(f"Created: {job.created_at.isoformat()}")
|
||||||
|
ui.label(f"Updated: {job.updated_at.isoformat()}")
|
||||||
|
|
||||||
|
if document is not None:
|
||||||
|
ui.label(f"Filename: {document.filename}")
|
||||||
|
ui.label(f"File path: {document.file_path}")
|
||||||
|
|
||||||
|
if transcript is None:
|
||||||
|
ui.label("Transcript not available yet.")
|
||||||
|
elif transcript.text:
|
||||||
|
ui.label("Transcript:")
|
||||||
|
ui.markdown(transcript.text)
|
||||||
|
elif transcript.error_detail:
|
||||||
|
ui.label("Failure detail:")
|
||||||
|
ui.label(transcript.error_detail)
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
"""Reusable jobs table rendering helpers."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from nicegui import ui
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class JobTableRow:
|
||||||
|
"""Read model consumed by the shared jobs table component."""
|
||||||
|
|
||||||
|
id: UUID
|
||||||
|
status: str
|
||||||
|
created_at: str
|
||||||
|
updated_at: str
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_rows(rows: Sequence[JobTableRow]) -> list[dict[str, str]]:
|
||||||
|
"""Convert typed rows into table-compatible dictionaries."""
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"id": str(row.id),
|
||||||
|
"status": row.status,
|
||||||
|
"created_at": row.created_at,
|
||||||
|
"updated_at": row.updated_at,
|
||||||
|
}
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def render_jobs_table(rows: Sequence[JobTableRow]) -> None:
|
||||||
|
"""Render jobs table and per-row detail links."""
|
||||||
|
if not rows:
|
||||||
|
ui.label("No jobs yet.")
|
||||||
|
return
|
||||||
|
|
||||||
|
serialized_rows = _serialize_rows(rows)
|
||||||
|
ui.table(
|
||||||
|
columns=[
|
||||||
|
{"name": "id", "label": "Job ID", "field": "id"},
|
||||||
|
{"name": "status", "label": "Status", "field": "status"},
|
||||||
|
{"name": "created_at", "label": "Created", "field": "created_at"},
|
||||||
|
{"name": "updated_at", "label": "Updated", "field": "updated_at"},
|
||||||
|
],
|
||||||
|
rows=serialized_rows,
|
||||||
|
row_key="id",
|
||||||
|
).classes("w-full")
|
||||||
|
|
||||||
|
with ui.column().classes("gap-1"):
|
||||||
|
for row in serialized_rows:
|
||||||
|
ui.link(f"Open {row['id']}", f"/jobs/{row['id']}")
|
||||||
@@ -1,140 +0,0 @@
|
|||||||
"""Jobs list and detail page registration."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from nicegui import ui
|
|
||||||
from sqlmodel import select
|
|
||||||
|
|
||||||
from transcription.db import get_session
|
|
||||||
from transcription.models import Document
|
|
||||||
from transcription.models import Job
|
|
||||||
from transcription.models import Transcript
|
|
||||||
from transcription.ui.error_presenter import show_error
|
|
||||||
from transcription.ui.error_presenter import summarize_error
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class JobView:
|
|
||||||
"""Read model for rendering job rows in the UI."""
|
|
||||||
|
|
||||||
id: UUID
|
|
||||||
status: str
|
|
||||||
created_at: str
|
|
||||||
updated_at: str
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_jobs() -> list[JobView]:
|
|
||||||
"""Return jobs for display in most-recent-first order."""
|
|
||||||
with get_session() as session:
|
|
||||||
jobs = session.exec(select(Job).order_by(Job.created_at.desc())).all()
|
|
||||||
return [
|
|
||||||
JobView(
|
|
||||||
id=job.id,
|
|
||||||
status=job.status.value,
|
|
||||||
created_at=job.created_at.isoformat(),
|
|
||||||
updated_at=job.updated_at.isoformat(),
|
|
||||||
)
|
|
||||||
for job in jobs
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_job_detail(job_id: UUID) -> tuple[Job | None, Document | None, Transcript | None]:
|
|
||||||
"""Return job, document, and transcript for detail view."""
|
|
||||||
with get_session() as session:
|
|
||||||
job = session.get(Job, job_id)
|
|
||||||
if job is None:
|
|
||||||
return None, None, None
|
|
||||||
document = session.get(Document, job.document_id)
|
|
||||||
transcript = session.exec(select(Transcript).where(Transcript.job_id == job.id)).first()
|
|
||||||
return job, document, transcript
|
|
||||||
|
|
||||||
|
|
||||||
def register_page() -> None:
|
|
||||||
"""Register jobs list and detail routes."""
|
|
||||||
|
|
||||||
@ui.page("/jobs")
|
|
||||||
def jobs_page() -> None:
|
|
||||||
ui.label("Transcription Jobs")
|
|
||||||
status = ui.label("Ready")
|
|
||||||
table_container = ui.column()
|
|
||||||
|
|
||||||
def render_table() -> None:
|
|
||||||
table_container.clear()
|
|
||||||
jobs = fetch_jobs()
|
|
||||||
with table_container:
|
|
||||||
if not jobs:
|
|
||||||
ui.label("No jobs yet.")
|
|
||||||
return
|
|
||||||
rows = [
|
|
||||||
{
|
|
||||||
"id": str(job.id),
|
|
||||||
"status": job.status,
|
|
||||||
"created_at": job.created_at,
|
|
||||||
"updated_at": job.updated_at,
|
|
||||||
}
|
|
||||||
for job in jobs
|
|
||||||
]
|
|
||||||
ui.table(
|
|
||||||
columns=[
|
|
||||||
{"name": "id", "label": "Job ID", "field": "id"},
|
|
||||||
{"name": "status", "label": "Status", "field": "status"},
|
|
||||||
{"name": "created_at", "label": "Created", "field": "created_at"},
|
|
||||||
{"name": "updated_at", "label": "Updated", "field": "updated_at"},
|
|
||||||
],
|
|
||||||
rows=rows,
|
|
||||||
row_key="id",
|
|
||||||
)
|
|
||||||
for row in rows:
|
|
||||||
ui.link(f"Open {row['id']}", f"/jobs/{row['id']}")
|
|
||||||
|
|
||||||
def refresh() -> None:
|
|
||||||
status.text = "Refreshing..."
|
|
||||||
try:
|
|
||||||
render_table()
|
|
||||||
status.text = "Refreshed"
|
|
||||||
except Exception as exc: # noqa: BLE001
|
|
||||||
status.text = f"Refresh failed: {summarize_error(exc, operation='jobs.refresh')}"
|
|
||||||
show_error(exc, title="Jobs refresh failed", operation="jobs.refresh")
|
|
||||||
|
|
||||||
ui.button("Refresh", on_click=refresh)
|
|
||||||
render_table()
|
|
||||||
ui.link("Back to upload", "/")
|
|
||||||
|
|
||||||
@ui.page("/jobs/{job_id}")
|
|
||||||
def job_detail_page(job_id: str) -> None:
|
|
||||||
ui.label("Job Detail")
|
|
||||||
try:
|
|
||||||
parsed_id = UUID(job_id)
|
|
||||||
except ValueError:
|
|
||||||
ui.label("Invalid job id")
|
|
||||||
ui.link("Back to jobs", "/jobs")
|
|
||||||
return
|
|
||||||
|
|
||||||
job, document, transcript = fetch_job_detail(parsed_id)
|
|
||||||
if job is None:
|
|
||||||
ui.label("Job not found")
|
|
||||||
ui.link("Back to jobs", "/jobs")
|
|
||||||
return
|
|
||||||
|
|
||||||
ui.label(f"Job ID: {job.id}")
|
|
||||||
ui.label(f"Status: {job.status.value}")
|
|
||||||
ui.label(f"Created: {job.created_at.isoformat()}")
|
|
||||||
ui.label(f"Updated: {job.updated_at.isoformat()}")
|
|
||||||
|
|
||||||
if document is not None:
|
|
||||||
ui.label(f"Filename: {document.filename}")
|
|
||||||
ui.label(f"File path: {document.file_path}")
|
|
||||||
|
|
||||||
if transcript is None:
|
|
||||||
ui.label("Transcript not available yet.")
|
|
||||||
elif transcript.text:
|
|
||||||
ui.label("Transcript:")
|
|
||||||
ui.markdown(transcript.text)
|
|
||||||
elif transcript.error_detail:
|
|
||||||
ui.label("Failure detail:")
|
|
||||||
ui.label(transcript.error_detail)
|
|
||||||
|
|
||||||
ui.link("Back to jobs", "/jobs")
|
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
"""Jobs list and detail page registration."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from nicegui import ui
|
||||||
|
from sqlmodel import desc
|
||||||
|
from sqlmodel import select
|
||||||
|
|
||||||
|
from transcription.db import get_session
|
||||||
|
from transcription.models import Document
|
||||||
|
from transcription.models import Job
|
||||||
|
from transcription.models import Transcript
|
||||||
|
from transcription.ui.components.error_presenter import show_error
|
||||||
|
from transcription.ui.components.error_presenter import summarize_error
|
||||||
|
from transcription.ui.components.job_detail import render_job_detail
|
||||||
|
from transcription.ui.components.job_table import JobTableRow
|
||||||
|
from transcription.ui.components.job_table import render_jobs_table
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_jobs() -> list[JobTableRow]:
|
||||||
|
"""Return jobs for display in most-recent-first order."""
|
||||||
|
async with get_session() as session:
|
||||||
|
jobs = (await session.exec(select(Job).order_by(desc(Job.created_at)))).all()
|
||||||
|
return [
|
||||||
|
JobTableRow(
|
||||||
|
id=job.id,
|
||||||
|
status=job.status.value,
|
||||||
|
created_at=job.created_at.isoformat(),
|
||||||
|
updated_at=job.updated_at.isoformat(),
|
||||||
|
)
|
||||||
|
for job in jobs
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_job_detail(job_id: UUID) -> tuple[Job | None, Document | None, Transcript | None]:
|
||||||
|
"""Return job, document, and transcript for detail view."""
|
||||||
|
async with get_session() as session:
|
||||||
|
job = await session.get(Job, job_id)
|
||||||
|
if job is None:
|
||||||
|
return None, None, None
|
||||||
|
document = await session.get(Document, job.document_id)
|
||||||
|
transcript = (await session.exec(select(Transcript).where(Transcript.job_id == job.id))).first()
|
||||||
|
return job, document, transcript
|
||||||
|
|
||||||
|
|
||||||
|
def register_page() -> None:
|
||||||
|
"""Register jobs list and detail routes."""
|
||||||
|
|
||||||
|
@ui.page("/jobs")
|
||||||
|
async def jobs_page() -> None:
|
||||||
|
ui.label("Transcription Jobs")
|
||||||
|
status = ui.label("Ready")
|
||||||
|
|
||||||
|
@ui.refreshable
|
||||||
|
async def render_table() -> None:
|
||||||
|
jobs = await fetch_jobs()
|
||||||
|
render_jobs_table(jobs)
|
||||||
|
|
||||||
|
async def refresh() -> None:
|
||||||
|
status.text = "Refreshing..."
|
||||||
|
try:
|
||||||
|
await render_table.refresh()
|
||||||
|
status.text = "Refreshed"
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
status.text = f"Refresh failed: {summarize_error(exc, operation='jobs.refresh')}"
|
||||||
|
show_error(exc, title="Jobs refresh failed", operation="jobs.refresh")
|
||||||
|
|
||||||
|
ui.button("Refresh", on_click=refresh)
|
||||||
|
await render_table()
|
||||||
|
ui.link("Back to upload", "/")
|
||||||
|
|
||||||
|
@ui.page("/jobs/{job_id}")
|
||||||
|
async def job_detail_page(job_id: str) -> None:
|
||||||
|
ui.label("Job Detail")
|
||||||
|
try:
|
||||||
|
parsed_id = UUID(job_id)
|
||||||
|
except ValueError:
|
||||||
|
ui.label("Invalid job id")
|
||||||
|
ui.link("Back to jobs", "/jobs")
|
||||||
|
return
|
||||||
|
|
||||||
|
job, document, transcript = await fetch_job_detail(parsed_id)
|
||||||
|
if job is None:
|
||||||
|
ui.label("Job not found")
|
||||||
|
ui.link("Back to jobs", "/jobs")
|
||||||
|
return
|
||||||
|
|
||||||
|
render_job_detail(job=job, document=document, transcript=transcript)
|
||||||
|
|
||||||
|
ui.link("Back to jobs", "/jobs")
|
||||||
@@ -10,8 +10,8 @@ from nicegui.events import UploadEventArguments
|
|||||||
from transcription.services.upload import UploadError
|
from transcription.services.upload import UploadError
|
||||||
from transcription.services.upload import UploadJobResult
|
from transcription.services.upload import UploadJobResult
|
||||||
from transcription.services.upload import create_upload_job
|
from transcription.services.upload import create_upload_job
|
||||||
from transcription.ui.error_presenter import show_error
|
from transcription.ui.components.error_presenter import show_error
|
||||||
from transcription.ui.error_presenter import summarize_error
|
from transcription.ui.components.error_presenter import summarize_error
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -27,9 +27,9 @@ def accepted_upload_types() -> str:
|
|||||||
return ".jpg,.jpeg,.png,.tif,.tiff,.pdf"
|
return ".jpg,.jpeg,.png,.tif,.tiff,.pdf"
|
||||||
|
|
||||||
|
|
||||||
def submit_upload(*, filename: str, file_bytes: bytes) -> UploadJobResult:
|
async def submit_upload(*, filename: str, file_bytes: bytes) -> UploadJobResult:
|
||||||
"""Create an upload job from incoming file data."""
|
"""Create an upload job from incoming file data."""
|
||||||
return create_upload_job(filename=filename, file_bytes=file_bytes)
|
return await create_upload_job(filename=filename, file_bytes=file_bytes)
|
||||||
|
|
||||||
|
|
||||||
def register_page() -> None:
|
def register_page() -> None:
|
||||||
@@ -49,7 +49,7 @@ def register_page() -> None:
|
|||||||
status_label.text = "Uploading..."
|
status_label.text = "Uploading..."
|
||||||
try:
|
try:
|
||||||
payload = await event.file.read()
|
payload = await event.file.read()
|
||||||
result = submit_upload(filename=event.file.name, file_bytes=payload)
|
result = await submit_upload(filename=event.file.name, file_bytes=payload)
|
||||||
state.message = f"Created job {result.job_id}"
|
state.message = f"Created job {result.job_id}"
|
||||||
status_label.text = state.message
|
status_label.text = state.message
|
||||||
ui.notify(state.message, type="positive")
|
ui.notify(state.message, type="positive")
|
||||||
+57
-34
@@ -2,16 +2,16 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import time
|
|
||||||
from datetime import UTC
|
from datetime import UTC
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from threading import Event
|
from threading import Event
|
||||||
|
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
from sqlalchemy.engine import Engine
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||||
from sqlmodel import Session
|
|
||||||
from sqlmodel import select
|
from sqlmodel import select
|
||||||
|
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||||
|
|
||||||
from transcription.config import Settings
|
from transcription.config import Settings
|
||||||
from transcription.config import get_settings
|
from transcription.config import get_settings
|
||||||
@@ -29,23 +29,23 @@ from transcription.services.transcription import transcribe_document_image
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def process_next_queued_job(*, session: Session | None = None, engine: Engine | None = None) -> bool:
|
async def process_next_queued_job(
|
||||||
|
*,
|
||||||
|
session: AsyncSession | None = None,
|
||||||
|
session_factory: async_sessionmaker[AsyncSession] | None = None,
|
||||||
|
) -> bool:
|
||||||
"""Process the next queued job and persist terminal outcome.
|
"""Process the next queued job and persist terminal outcome.
|
||||||
|
|
||||||
Returns True when a job was processed, False when no queued job exists.
|
Returns True when a job was processed, False when no queued job exists.
|
||||||
"""
|
"""
|
||||||
if session is None:
|
if session is None:
|
||||||
with get_session(engine=engine) as local_session:
|
async with get_session(session_factory=session_factory) as local_session:
|
||||||
return _process_next_queued_job(session=local_session)
|
return await _process_next_queued_job(session=local_session)
|
||||||
return _process_next_queued_job(session=session)
|
return await _process_next_queued_job(session=session)
|
||||||
|
|
||||||
|
|
||||||
def _process_next_queued_job(*, session: Session) -> bool:
|
async def _process_next_queued_job(*, session: AsyncSession) -> bool:
|
||||||
job = session.exec(
|
job = (await session.exec(select(Job).where(Job.status == JobStatus.QUEUED).order_by(Job.created_at))).first()
|
||||||
select(Job)
|
|
||||||
.where(Job.status == JobStatus.QUEUED)
|
|
||||||
.order_by(Job.created_at)
|
|
||||||
).first()
|
|
||||||
|
|
||||||
if job is None:
|
if job is None:
|
||||||
return False
|
return False
|
||||||
@@ -54,10 +54,10 @@ def _process_next_queued_job(*, session: Session) -> bool:
|
|||||||
job.status = JobStatus.PROCESSING
|
job.status = JobStatus.PROCESSING
|
||||||
job.updated_at = datetime.now(UTC)
|
job.updated_at = datetime.now(UTC)
|
||||||
session.add(job)
|
session.add(job)
|
||||||
session.commit()
|
await session.commit()
|
||||||
session.refresh(job)
|
await session.refresh(job)
|
||||||
|
|
||||||
document = session.get(Document, job.document_id)
|
document = await session.get(Document, job.document_id)
|
||||||
if document is None:
|
if document is None:
|
||||||
error = AppError(
|
error = AppError(
|
||||||
"Document not found",
|
"Document not found",
|
||||||
@@ -75,11 +75,11 @@ def _process_next_queued_job(*, session: Session) -> bool:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
result = transcribe_document_image(document.file_path)
|
result = transcribe_document_image(document.file_path)
|
||||||
_upsert_transcript(session=session, job_id=job.id, text=result.text, error_detail=None)
|
await _upsert_transcript(session=session, job_id=job.id, text=result.text, error_detail=None)
|
||||||
job.status = JobStatus.TRANSCRIBED
|
job.status = JobStatus.TRANSCRIBED
|
||||||
job.updated_at = datetime.now(UTC)
|
job.updated_at = datetime.now(UTC)
|
||||||
session.add(job)
|
session.add(job)
|
||||||
session.commit()
|
await session.commit()
|
||||||
logger.info(
|
logger.info(
|
||||||
"Job transcribed operation=worker.process_job job_id=%s document_id=%s provider=%s",
|
"Job transcribed operation=worker.process_job job_id=%s document_id=%s provider=%s",
|
||||||
job.id,
|
job.id,
|
||||||
@@ -90,7 +90,7 @@ def _process_next_queued_job(*, session: Session) -> bool:
|
|||||||
error = exc if isinstance(exc, AppError) else classify_unexpected_error(exc, operation="worker.process_job")
|
error = exc if isinstance(exc, AppError) else classify_unexpected_error(exc, operation="worker.process_job")
|
||||||
settings = _get_worker_settings()
|
settings = _get_worker_settings()
|
||||||
if _should_retry(job=job, error=error, settings=settings):
|
if _should_retry(job=job, error=error, settings=settings):
|
||||||
_requeue_for_retry(session=session, job=job, error=error, settings=settings)
|
await _requeue_for_retry(session=session, job=job, error=error, settings=settings)
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Job retried operation=worker.process_job job_id=%s document_id=%s retry_count=%s error_id=%s category=%s",
|
"Job retried operation=worker.process_job job_id=%s document_id=%s retry_count=%s error_id=%s category=%s",
|
||||||
job.id,
|
job.id,
|
||||||
@@ -100,7 +100,7 @@ def _process_next_queued_job(*, session: Session) -> bool:
|
|||||||
error.category.value,
|
error.category.value,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
_finalize_failed_job(session=session, job=job, error=error)
|
await _finalize_failed_job(session=session, job=job, error=error)
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"Job failed operation=worker.process_job job_id=%s document_id=%s error_id=%s category=%s",
|
"Job failed operation=worker.process_job job_id=%s document_id=%s error_id=%s category=%s",
|
||||||
job.id,
|
job.id,
|
||||||
@@ -112,16 +112,18 @@ def _process_next_queued_job(*, session: Session) -> bool:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def _upsert_transcript(*, session: Session, job_id, text: str | None, error_detail: str | None) -> Transcript:
|
async def _upsert_transcript(
|
||||||
transcript = session.exec(select(Transcript).where(Transcript.job_id == job_id)).first()
|
*, session: AsyncSession, job_id, text: str | None, error_detail: str | None
|
||||||
|
) -> Transcript:
|
||||||
|
transcript = (await session.exec(select(Transcript).where(Transcript.job_id == job_id))).first()
|
||||||
if transcript is None:
|
if transcript is None:
|
||||||
transcript = Transcript(job_id=job_id)
|
transcript = Transcript(job_id=job_id)
|
||||||
|
|
||||||
transcript.text = text
|
transcript.text = text
|
||||||
transcript.error_detail = error_detail
|
transcript.error_detail = error_detail
|
||||||
session.add(transcript)
|
session.add(transcript)
|
||||||
session.commit()
|
await session.commit()
|
||||||
session.refresh(transcript)
|
await session.refresh(transcript)
|
||||||
return transcript
|
return transcript
|
||||||
|
|
||||||
|
|
||||||
@@ -136,32 +138,53 @@ def _should_retry(*, job: Job, error: AppError, settings: Settings) -> bool:
|
|||||||
return error.retriable and job.retry_count < settings.worker_max_retries
|
return error.retriable and job.retry_count < settings.worker_max_retries
|
||||||
|
|
||||||
|
|
||||||
def _requeue_for_retry(*, session: Session, job: Job, error: AppError, settings: Settings) -> None:
|
async def _requeue_for_retry(*, session: AsyncSession, job: Job, error: AppError, settings: Settings) -> None:
|
||||||
_upsert_transcript(session=session, job_id=job.id, text=None, error_detail=format_error_detail(error))
|
await _upsert_transcript(session=session, job_id=job.id, text=None, error_detail=format_error_detail(error))
|
||||||
job.retry_count += 1
|
job.retry_count += 1
|
||||||
job.status = JobStatus.QUEUED
|
job.status = JobStatus.QUEUED
|
||||||
job.updated_at = datetime.now(UTC)
|
job.updated_at = datetime.now(UTC)
|
||||||
session.add(job)
|
session.add(job)
|
||||||
session.commit()
|
await session.commit()
|
||||||
if settings.worker_retry_backoff_seconds > 0:
|
if settings.worker_retry_backoff_seconds > 0:
|
||||||
time.sleep(settings.worker_retry_backoff_seconds)
|
await asyncio.sleep(settings.worker_retry_backoff_seconds)
|
||||||
|
|
||||||
|
|
||||||
def _finalize_failed_job(*, session: Session, job: Job, error: AppError) -> None:
|
async def _finalize_failed_job(*, session: AsyncSession, job: Job, error: AppError) -> None:
|
||||||
_upsert_transcript(session=session, job_id=job.id, text=None, error_detail=format_error_detail(error))
|
await _upsert_transcript(session=session, job_id=job.id, text=None, error_detail=format_error_detail(error))
|
||||||
job.status = JobStatus.FAILED
|
job.status = JobStatus.FAILED
|
||||||
job.updated_at = datetime.now(UTC)
|
job.updated_at = datetime.now(UTC)
|
||||||
session.add(job)
|
session.add(job)
|
||||||
session.commit()
|
await session.commit()
|
||||||
|
|
||||||
|
|
||||||
def run_worker_loop(*, engine: Engine | None = None, stop_event: Event | None = None, poll_interval_seconds: float = 1.0) -> None:
|
async def _run_worker_loop_async(
|
||||||
|
*,
|
||||||
|
session_factory: async_sessionmaker[AsyncSession] | None = None,
|
||||||
|
stop_event: Event | None = None,
|
||||||
|
poll_interval_seconds: float = 1.0,
|
||||||
|
) -> None:
|
||||||
"""Run worker polling loop until stop_event is set."""
|
"""Run worker polling loop until stop_event is set."""
|
||||||
while True:
|
while True:
|
||||||
if stop_event is not None and stop_event.is_set():
|
if stop_event is not None and stop_event.is_set():
|
||||||
logger.info("Worker stop event received")
|
logger.info("Worker stop event received")
|
||||||
return
|
return
|
||||||
|
|
||||||
processed = process_next_queued_job(engine=engine)
|
processed = await process_next_queued_job(session_factory=session_factory)
|
||||||
if not processed:
|
if not processed:
|
||||||
time.sleep(poll_interval_seconds)
|
await asyncio.sleep(poll_interval_seconds)
|
||||||
|
|
||||||
|
|
||||||
|
def run_worker_loop(
|
||||||
|
*,
|
||||||
|
session_factory: async_sessionmaker[AsyncSession] | None = None,
|
||||||
|
stop_event: Event | None = None,
|
||||||
|
poll_interval_seconds: float = 1.0,
|
||||||
|
) -> None:
|
||||||
|
"""Synchronous thread entrypoint that runs the async worker loop."""
|
||||||
|
asyncio.run(
|
||||||
|
_run_worker_loop_async(
|
||||||
|
session_factory=session_factory,
|
||||||
|
stop_event=stop_event,
|
||||||
|
poll_interval_seconds=poll_interval_seconds,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user