bbchops/transcription
1
0
Fork 0
generated from john/python-template
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Jobs: jobs still stuck in queue. Fixes from testing.

Browse Source
This commit is contained in:
Jim Lancaster
2026-08-04 18:09:31 -05:00
parent 6c6589d8ff
commit 271633d1d5
12 changed files with 278 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/transcription/services/documents.py
+2 -6
View File
@@ -217,12 +217,8 @@ class DocumentService(ServiceBase):
suggestion="Verify the person id and retry.",
)
if existing.document_people:
raise PersonDeleteBlockedError(
"Person delete blocked by linked documents",
category=ErrorCategory.VALIDATION,
suggestion="Remove linked DocumentPerson records first, then retry deletion.",
)
for link in list(existing.document_people):
await _session.delete(link)
await _session.delete(existing)
await self._finalize(session=_session, caller_session=session)
src/transcription/services/store.py
+64 -19
View File
@@ -41,6 +41,15 @@ class JobCreateResult:
source_ids: tuple[UUID, ...]
@dataclass(frozen=True)
class PendingStoredUpload:
"""Pre-staged upload artifact tied to a source id."""
source_id: UUID
original_filename: str
stored_path: Path
async def create_upload_job(
*,
filename: str,
@@ -50,14 +59,20 @@ async def create_upload_job(
) -> UploadJobResult:
"""Create upload-backed document and queued job records."""
runtime_settings = settings or get_settings()
document_id = uuid4()
source_id = uuid4()
stored_path = store_file(
filename=filename,
file_bytes=file_bytes,
settings=runtime_settings,
relative_directory=Path("documents") / str(document_id),
filename_stem=str(source_id),
)
try:
document, job = await _create_upload_records(
session=session,
document_id=document_id,
source_id=source_id,
original_filename=filename,
stored_path=stored_path,
)
@@ -99,15 +114,19 @@ async def create_job_for_document(
runtime_settings = settings or get_settings()
sorted_uploads = sorted(uploads, key=lambda item: Path(item[0]).name.casefold())
stored_uploads: list[tuple[str, Path]] = []
stored_uploads: list[PendingStoredUpload] = []
for filename, file_bytes in sorted_uploads:
source_id = uuid4()
stored_uploads.append(
(
filename,
store_file(
PendingStoredUpload(
source_id=source_id,
original_filename=filename,
stored_path=store_file(
filename=filename,
file_bytes=file_bytes,
settings=runtime_settings,
relative_directory=Path("documents") / str(document_id),
filename_stem=str(source_id),
),
)
)
@@ -122,8 +141,8 @@ async def create_job_for_document(
prompt_name=prompt_name,
)
except Exception as exc:
for _, stored_path in stored_uploads:
_best_effort_delete(stored_path)
for upload in stored_uploads:
_best_effort_delete(upload.stored_path)
raise UploadError(
"Failed to create job records from uploads",
category=ErrorCategory.INFRA_TRANSIENT,
@@ -142,10 +161,13 @@ async def create_job_for_document(
async def _create_upload_records(
*,
session: AsyncSession,
document_id: UUID,
source_id: UUID,
original_filename: str,
stored_path: Path,
) -> tuple[Document, Job]:
document = Document(
id=document_id,
name=Path(original_filename).name,
)
session.add(document)
@@ -156,6 +178,7 @@ async def _create_upload_records(
await session.flush()
source = Source(
id=source_id,
document_id=document.id,
page_number=1,
upload_name=Path(original_filename).name,
@@ -183,7 +206,7 @@ async def _create_job_for_document_records(
*,
session: AsyncSession,
document_id: UUID,
stored_uploads: Sequence[tuple[str, Path]],
stored_uploads: Sequence[PendingStoredUpload],
provider: str | None,
model: str | None,
prompt_name: str | None,
@@ -211,13 +234,14 @@ async def _create_job_for_document_records(
await session.flush()
source_ids: list[UUID] = []
for page_offset, (original_filename, stored_path) in enumerate(stored_uploads):
for page_offset, upload in enumerate(stored_uploads):
source = Source(
id=upload.source_id,
document_id=document_id,
page_number=next_page_number + page_offset,
upload_name=Path(original_filename).name,
filename=stored_path.name,
file_path=str(stored_path),
upload_name=Path(upload.original_filename).name,
filename=upload.stored_path.name,
file_path=str(upload.stored_path),
)
session.add(source)
await session.flush()
@@ -244,22 +268,41 @@ def _best_effort_delete(path: Path) -> None:
logger.warning("Failed to clean up upload file after DB error: %s", path)
def store_file(*, filename: str, file_bytes: bytes, settings: Settings | None = None) -> Path:
def store_file(
*,
filename: str,
file_bytes: bytes,
settings: Settings | None = None,
relative_directory: Path | None = None,
filename_stem: str | None = None,
) -> Path:
"""Persist an uploaded file to the configured upload directory."""
runtime_settings = settings or get_settings()
_validate_upload(filename=filename, file_bytes=file_bytes, supported_extensions=SUPPORTED_UPLOAD_EXTENSIONS)
return _store_file_bytes(filename=filename, file_bytes=file_bytes, settings=runtime_settings)
return _store_file_bytes(
filename=filename,
file_bytes=file_bytes,
settings=runtime_settings,
relative_directory=relative_directory,
filename_stem=filename_stem,
)
def store_person_portrait(*, filename: str, file_bytes: bytes, settings: Settings | None = None) -> Path:
"""Persist a portrait upload under uploads/portraits/person."""
def store_person_portrait(
*,
person_id: UUID,
filename: str,
file_bytes: bytes,
settings: Settings | None = None,
) -> Path:
"""Persist a portrait upload under persons/<person_id>."""
runtime_settings = settings or get_settings()
_validate_upload(filename=filename, file_bytes=file_bytes, supported_extensions=SUPPORTED_PORTRAIT_EXTENSIONS)
return _store_file_bytes(
filename=filename,
file_bytes=file_bytes,
settings=runtime_settings,
relative_directory=Path("portraits") / "person",
relative_directory=Path("persons") / str(person_id),
)
@@ -269,12 +312,13 @@ def _store_file_bytes(
file_bytes: bytes,
settings: Settings,
relative_directory: Path | None = None,
filename_stem: str | None = None,
) -> Path:
upload_dir = settings.upload_dir
target_dir = upload_dir if relative_directory is None else upload_dir / relative_directory
target_dir.mkdir(parents=True, exist_ok=True)
stored_name = _build_stored_filename(filename)
stored_name = _build_stored_filename(filename=filename, filename_stem=filename_stem)
stored_path = target_dir / stored_name
try:
@@ -315,7 +359,8 @@ def _validate_upload(*, filename: str, file_bytes: bytes, supported_extensions:
)
def _build_stored_filename(filename: str) -> str:
def _build_stored_filename(*, filename: str, filename_stem: str | None = None) -> str:
safe_name = Path(filename).name
suffix = Path(safe_name).suffix.lower()
return f"{uuid4()}{suffix}"
stem = filename_stem or str(uuid4())
return f"{stem}{suffix}"
src/transcription/ui/pages/people_page.py
+20 -22
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
from datetime import date
from urllib.parse import quote
from uuid import UUID
from uuid import uuid4
from fastapi import Request
from nicegui import ui
@@ -15,7 +16,6 @@ from transcription.errors import ErrorCategory
from transcription.services.documents import (
DocumentError,
DocumentService,
PersonDeleteBlockedError,
)
from transcription.services.store import UploadError, store_person_portrait
from transcription.ui.components.app_shell import render_navigation_header
@@ -44,11 +44,12 @@ def _parse_optional_date(value: str | None, *, label: str) -> date | None:
raise ValueError(f"{label} must use YYYY-MM-DD.") from exc
def _bind_portrait_file_picker(portrait_path_input: ui.input, *, settings: Settings) -> None:
def _bind_portrait_file_picker(portrait_path_input: ui.input, *, settings: Settings, person_id: UUID) -> None:
async def on_portrait_selected(event) -> None:
payload = await event.file.read()
try:
stored_path = store_person_portrait(
person_id=person_id,
filename=event.file.name,
file_bytes=payload,
settings=settings,
@@ -73,7 +74,8 @@ def _bind_portrait_file_picker(portrait_path_input: ui.input, *, settings: Setti
auto_upload=True,
label="Choose portrait file",
).props('accept=".jpg,.jpeg,.png,.gif,.webp,.bmp,.tif,.tiff"').classes("w-full")
ui.label("Portraits are stored under uploads/portraits/person.").classes("text-xs ui-text-muted")
portrait_dir = settings.upload_dir / "persons" / str(person_id)
ui.label(f"Portraits are stored under {portrait_dir}.").classes("text-xs ui-text-muted")
def _resolve_portrait_src(path: str | None) -> str | None:
@@ -145,6 +147,7 @@ def register_page() -> None: # noqa: PLR0915
apply_archival_theme()
people_service = DocumentService(session_factory=session_factory)
render_navigation_header(current_path="/people")
draft_person_id = uuid4()
with ui.column().classes("w-full max-w-4xl mx-auto p-4 gap-4"):
page_header("Create Person Record", subtitle="Full name is required.")
@@ -167,7 +170,11 @@ def register_page() -> None: # noqa: PLR0915
biography_input = ui.textarea(label="Biography").props("outlined bg-white autogrow").classes("w-full")
portrait_path_input = ui.input(label="Portrait path").props("outlined bg-white").classes("w-full")
_bind_portrait_file_picker(portrait_path_input, settings=_resolve_runtime_settings(request))
_bind_portrait_file_picker(
portrait_path_input,
settings=_resolve_runtime_settings(request),
person_id=draft_person_id,
)
async def submit_create() -> None:
full_name = (full_name_input.value or "").strip()
@@ -183,6 +190,7 @@ def register_page() -> None: # noqa: PLR0915
return
candidate = Person(
id=draft_person_id,
full_name=full_name,
display_name=(display_name_input.value or "").strip() or None,
maiden_name=(maiden_name_input.value or "").strip() or None,
@@ -357,7 +365,11 @@ def register_page() -> None: # noqa: PLR0915
portrait_path_input = (
ui.input(label="Portrait path", value=person.portrait_path or "").props("outlined bg-white").classes("w-full")
)
_bind_portrait_file_picker(portrait_path_input, settings=_resolve_runtime_settings(request))
_bind_portrait_file_picker(
portrait_path_input,
settings=_resolve_runtime_settings(request),
person_id=person.id,
)
async def submit_edit() -> None:
full_name = (full_name_input.value or "").strip()
@@ -431,29 +443,15 @@ def register_page() -> None: # noqa: PLR0915
ui.label(f"Person: {person.full_name}").classes("text-sm font-semibold ui-text-primary")
if person.document_people:
ui.label("Delete is blocked because linked documents exist.").classes("text-xs text-red-800 font-bold mt-2")
ui.label(f"Linked documents: {len(person.document_people)}").classes("text-xs ui-text-muted")
ui.label("Remove document links first, then retry deletion.").classes("text-xs ui-text-muted italic")
with ui.row().classes("w-full items-center gap-2 mt-4"):
ui.button(
"Back to Person",
on_click=lambda: ui.navigate.to(f"/people/{person.id}"),
icon="arrow_back",
).classes("ui-btn-primary text-xs")
ui.button("Go to Documents", on_click=lambda: ui.navigate.to("/documents"), icon="description").props(
"flat text-xs"
)
return
ui.label(
f"This will also remove {len(person.document_people)} linked document relationship(s)."
).classes("text-xs text-red-800 font-bold mt-2")
ui.label("This action permanently deletes the person record.").classes("text-xs text-red-800 font-medium")
async def submit_delete() -> None:
try:
await people_service.delete_person(person)
except PersonDeleteBlockedError as exc:
ui.notify(exc.message, type="warning")
ui.navigate.to(f"/people/{person.id}/delete")
return
except DocumentError as exc:
if exc.category == ErrorCategory.NOT_FOUND:
ui.notify("Person not found.", type="warning")
src/transcription/ui/pages/sources_page.py
+11
View File
@@ -176,6 +176,17 @@ def register_page() -> None:
source.date_revised.isoformat() if source.date_revised else "Not revised",
)
with archival_card(title="Job Source Outcomes"):
if not source.job_sources:
ui.label("No job-source execution records found for this source.").classes("text-xs ui-text-muted")
else:
for job_source in sorted(source.job_sources, key=lambda item: item.executed_at, reverse=True):
with ui.column().classes("w-full gap-1 p-2 ui-row-surface rounded"):
metadata_row("Job ID:", str(job_source.job_id))
metadata_row("Status:", job_source.status.value)
metadata_row("Executed At:", job_source.executed_at.isoformat())
metadata_row("Error Detail:", job_source.error_detail or "None")
with archival_card(title="Automated Raw Transcription"):
ui.textarea(value=_source_transcription_text(source) or "").props("outlined autogrow readonly bg-white").classes(
"w-full text-xs font-mono"
src/transcription/worker.py
+9 -2
View File
@@ -152,8 +152,15 @@ async def run_worker_loop(
wake_event.clear()
processed_any = False
while await process_next_queued_job(session_factory=session_factory):
processed_any = True
while True:
with handle_worker_exceptions(operation="worker.process_next_queued_job"):
processed = await process_next_queued_job(session_factory=session_factory)
if not processed:
break
processed_any = True
continue
break
if wake_event is None and not processed_any:
await asyncio.sleep(poll_interval_seconds)