Continue GC code review and cleanup

This commit is contained in:
Jim Lancaster
2026-08-11 16:42:09 -05:00
parent 8d5aec4301
commit b8be27f0c9
16 changed files with 363 additions and 161 deletions
+11 -6
View File
@@ -24,7 +24,10 @@ from .db import create_all
from .db import dispose_database_runtime
from .db import initialize_database_runtime
from .services import ServiceBundle
from .services.documents import DocumentService
from .services.jobs import JobService
from .services.people import PeopleService
from .services.sources import SourceService
from .ui import register_pages
from .worker import worker_consumer_lifespan
@@ -36,8 +39,14 @@ async def _lifespan(app: FastAPI):
settings = getattr(app.state, "settings", None) or get_settings()
configure_logging(settings)
app.state.settings = settings
app.state.services = ServiceBundle()
app.state.runtime = initialize_database_runtime(settings=settings)
session_factory = app.state.runtime.session_factory
app.state.services = ServiceBundle(
documents=DocumentService(session_factory=session_factory, settings=settings),
sources=SourceService(session_factory=session_factory, settings=settings),
jobs=JobService(session_factory=session_factory, settings=settings),
people=PeopleService(session_factory=session_factory, settings=settings),
)
if settings.should_bootstrap_schema:
await create_all(engine=app.state.runtime.engine)
@@ -93,12 +102,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
async def ui_redirect() -> RedirectResponse:
return RedirectResponse(url="/ui/homepage", status_code=status.HTTP_307_TEMPORARY_REDIRECT)
@app.get("/healthz")
def health() -> dict[str, str]:
return {"status": "ok"}
register_error_handlers(app)
register_pages(app)
app.include_router(health_router)
app.include_router(v4_documents_router)
register_pages(app)
return app
+12 -6
View File
@@ -1,5 +1,4 @@
import logging
from contextvars import ContextVar
from dataclasses import dataclass
from sqlalchemy.ext.asyncio import AsyncEngine
@@ -23,21 +22,28 @@ class DatabaseRuntime:
session_factory: async_sessionmaker[AsyncSession]
_runtime: ContextVar[DatabaseRuntime | None] = ContextVar("database_runtime", default=None)
_runtime: DatabaseRuntime | None = None
def get_database_runtime() -> DatabaseRuntime | None:
"""Return the process-owned database runtime."""
return _runtime
async def dispose_database_runtime() -> None:
"""Dispose lifespan-owned async database resources."""
runtime = _runtime.get()
global _runtime
runtime = _runtime
if runtime is None:
return
await runtime.engine.dispose()
_runtime.set(None)
_runtime = None
def initialize_database_runtime(*, settings: Settings | None = None) -> DatabaseRuntime:
"""Initialize lifespan-owned async DB resources once per process."""
runtime = _runtime.get()
global _runtime
runtime = _runtime
if runtime is not None:
return runtime
@@ -46,6 +52,6 @@ def initialize_database_runtime(*, settings: Settings | None = None) -> Database
engine = get_engine(database_url)
session_factory = get_session_factory(database_url)
runtime = DatabaseRuntime(engine=engine, session_factory=session_factory)
_runtime.set(runtime)
_runtime = runtime
logger.debug("Initialized async database runtime for database_url=%s", engine.url)
return runtime
+6
View File
@@ -33,6 +33,12 @@ def resolve_session_factory(
) -> SessionFactory:
if database_url is not None:
return get_session_factory(database_url)
if settings is None:
from .runtime import get_database_runtime
runtime = get_database_runtime()
if runtime is not None:
return runtime.session_factory
return get_session_factory(get_database_url(settings or get_settings()))
+1 -1
View File
@@ -23,8 +23,8 @@ class TranscriptionResult:
text: str
provider: str
prompt_name: str
model: str
prompt_name: str | None = None
prompt_hash: str | None = None
system_prompt: str | None = None
user_prompt: str | None = None
+18 -3
View File
@@ -95,7 +95,7 @@ class OpenRouterTranscriptionProvider:
return TranscriptionResult(
text=text,
provider="openrouter",
prompt_name="",
prompt_name=None,
prompt_hash=None,
system_prompt=None,
user_prompt=prompt_text,
@@ -158,7 +158,8 @@ class OpenRouterTranscriptionProvider:
if callable(serializer):
try:
return self._to_json_compatible(serializer())
except Exception: # noqa: BLE001
except Exception as exc: # noqa: BLE001
logger.debug("OpenRouter response serializer %s failed: %s", method_name, exc)
continue
object_dict = getattr(value, "__dict__", None)
@@ -182,13 +183,27 @@ class OpenRouterTranscriptionProvider:
) -> OpenRouterRequest:
image_b64 = base64.b64encode(image_bytes).decode("ascii")
data_url = f"data:{mime_type};base64,{image_b64}"
media_content: dict[str, Any]
if mime_type == "application/pdf":
media_content = {
"type": "file",
"file": {
"filename": "source.pdf",
"file_data": data_url,
},
}
else:
media_content = {
"type": "image_url",
"image_url": {"url": data_url},
}
messages: list[dict[str, Any]] = [
{
"role": "user",
"content": [
{"type": "text", "text": prompt_text},
{"type": "image_url", "image_url": {"url": data_url}},
media_content,
],
}
]
+3 -2
View File
@@ -23,9 +23,10 @@ class ServiceBase(ABC):
self,
session_factory: async_sessionmaker[AsyncSession] | None = None,
queue: asyncio.Queue | None = None,
settings: Settings | None = None,
):
self.settings = get_settings()
self.session_factory = session_factory or resolve_session_factory()
self.settings = settings or get_settings()
self.session_factory = session_factory or resolve_session_factory(settings=self.settings)
self.queue = queue or asyncio.Queue()
@asynccontextmanager
+19 -7
View File
@@ -7,13 +7,13 @@ from sqlalchemy.orm import selectinload
from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession
from ..errors import AppError
from ..errors import ErrorCategory
from ..db.models import Job
from ..db.models import JobSource
from ..db.models import JobSourceStatus
from ..db.models import JobStatus
from ..db.models import Source
from ..errors import AppError
from ..errors import ErrorCategory
from .base import ServiceBase
@@ -29,6 +29,10 @@ class JobResubmitBlockedError(AppError):
"""Raised when a job resubmit operation is blocked by lifecycle policy."""
class JobNotFoundError(AppError):
"""Raised when a requested Job does not exist."""
class JobService(ServiceBase):
"""Thin service class for managing jobs in the database."""
@@ -61,7 +65,7 @@ class JobService(ServiceBase):
)
job = (await _session.exec(query)).first()
if job is None:
raise ValueError(f"Job with id {job_id} not found")
raise self._not_found(job_id)
return job
async def update_job(self, job: Job, session: AsyncSession | None = None) -> Job:
@@ -148,7 +152,7 @@ class JobService(ServiceBase):
)
job = (await _session.exec(query)).first()
if job is None:
raise ValueError(f"Job with id {job_id} not found")
raise self._not_found(job_id)
job.status = status
if retry_count_increment:
job.retry_count += retry_count_increment
@@ -216,7 +220,7 @@ class JobService(ServiceBase):
)
job = (await _session.exec(query)).first()
if job is None:
raise ValueError(f"Job with id {job_id} not found")
raise self._not_found(job_id)
if job.status == JobStatus.PROCESSING:
raise JobDeleteBlockedError(
@@ -244,7 +248,7 @@ class JobService(ServiceBase):
)
job = (await _session.exec(query)).first()
if job is None:
raise ValueError(f"Job with id {job_id} not found")
raise self._not_found(job_id)
if job.status in {JobStatus.TRANSCRIBED, JobStatus.COMPLETED}:
raise JobCancelBlockedError(
@@ -283,7 +287,7 @@ class JobService(ServiceBase):
)
job = (await _session.exec(query)).first()
if job is None:
raise ValueError(f"Job with id {job_id} not found")
raise self._not_found(job_id)
if job.status == JobStatus.PROCESSING:
raise JobResubmitBlockedError(
@@ -314,3 +318,11 @@ class JobService(ServiceBase):
await self._finalize(session=_session, caller_session=session, refresh=(job,))
return len(candidates)
@staticmethod
def _not_found(job_id: UUID) -> JobNotFoundError:
return JobNotFoundError(
f"Job with id {job_id} not found",
category=ErrorCategory.NOT_FOUND,
suggestion="Verify the Job id and retry.",
)
+6 -2
View File
@@ -81,8 +81,12 @@ class SourceService(ServiceBase):
provider: TranscriptionProvider
def __init__(self, session_factory: async_sessionmaker[AsyncSession] | None = None):
super().__init__(session_factory=session_factory)
def __init__(
self,
session_factory: async_sessionmaker[AsyncSession] | None = None,
settings: Settings | None = None,
):
super().__init__(session_factory=session_factory, settings=settings)
self.provider = get_transcription_provider(settings=self.settings)
async def create_source(self, source: Source, *, session: AsyncSession | None = None) -> Source:
-22
View File
@@ -9,7 +9,6 @@ from contextlib import asynccontextmanager
from contextlib import contextmanager
from contextlib import suppress
from typing import Protocol
from uuid import UUID
from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlmodel.ext.asyncio.session import AsyncSession
@@ -23,7 +22,6 @@ from .services.documents import DocumentService
from .services.jobs import JobService
from .services.people import PeopleService
from .services.sources import SourceService
from .services.workflows import advance_job
from .services.workflows import process_next_queued_job as process_next_queued_job_workflow
logger = logging.getLogger(__name__)
@@ -96,19 +94,6 @@ async def worker_consumer_lifespan(
await worker_task
async def queue_consumer_loop(queue: asyncio.Queue[UUID], stop_event: asyncio.Event):
"""Main worker loop that consumes jobs from the queue and processes them.
The queue contains Job UUIDs whose Document and Source records already exist.
"""
service = JobService()
while not stop_event.is_set():
with handle_worker_exceptions():
async with _get_queue_item(queue) as job_id:
job = await service.read_job(job_id)
asyncio.create_task(advance_job(job=job, services=ServiceBundle()))
@contextmanager
def handle_worker_exceptions(operation: str = "worker.loop"):
"""Context manager to log and suppress exceptions in the worker loop."""
@@ -123,13 +108,6 @@ def handle_worker_exceptions(operation: str = "worker.loop"):
)
@asynccontextmanager
async def _get_queue_item(queue: asyncio.Queue[UUID]) -> AsyncGenerator[UUID]:
"""Context manager to enqueue a job and ensure it is marked done."""
yield await queue.get()
queue.task_done()
async def run_worker_loop(
*,
session_factory: async_sessionmaker[AsyncSession] | None = None,