bbchops/transcription
1
0
Fork 0
generated from john/python-template
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

V4.7 Phase 2: Evidence Model Simplification (part 2)

Browse Source
This commit is contained in:
zoltan57
2026-08-18 15:31:33 -05:00
parent 7285a87dfb
commit 11097b9cfe
15 changed files with 312 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/transcription/db/models.py
+41 -21
View File
@@ -70,6 +70,7 @@ class JobSourceStatus(StrEnum):
PENDING = "pending"
TRANSCRIBED = "transcribed"
FAILED = "failed"
CANCELLED = "cancelled"
class JobPurpose(StrEnum):
@@ -269,15 +270,6 @@ class Job(SQLModel, table=True):
return "unknown"
@property
def error_detail(self) -> str | None:
"""Return the first available source-level error detail for the job."""
for job_source in _loaded_attribute(self, "job_sources") or ():
if job_source.error_detail:
return job_source.error_detail
return None
class Source(SQLModel, table=True):
"""A document source image or PDF page."""
@@ -321,10 +313,21 @@ class Source(SQLModel, table=True):
@property
def latest_job_source(self) -> Optional["JobSource"]:
"""Return the most recent job execution record for this source."""
if not self.job_sources:
return None
return max(self.job_sources, key=lambda js: js.executed_at)
"""Return the most recent job execution record for this source.
``JobSource`` carries no timestamp of its own, so recency is the parent
job's creation time. ``(job_id, source_id)`` is unique per source, so
this is exactly "the most recent job that included this page".
"""
job_sources = _loaded_attribute(self, "job_sources") or ()
dated = [
(job, job_source)
for job_source in job_sources
if (job := _loaded_attribute(job_source, "job")) is not None
]
if dated:
return max(dated, key=lambda pair: pair[0].date_created)[1]
return job_sources[0] if job_sources else None
@property
def latest_status(self) -> JobSourceStatus | None:
@@ -334,9 +337,19 @@ class Source(SQLModel, table=True):
@property
def latest_error_detail(self) -> str | None:
"""Return the error detail from the latest job run, if present."""
"""Return the error detail of the latest attempt on the latest job run.
Failure detail lives on ``ExecutionAttempt``; ``JobSource`` records only
which page a job is working on and how far it got.
"""
latest = self.latest_job_source
return latest.error_detail if latest else None
if latest is None:
return None
attempts = _loaded_attribute(latest, "execution_attempts") or ()
for attempt in sorted(attempts, key=lambda item: item.attempt_number, reverse=True):
if attempt.error_detail:
return attempt.error_detail
return None
@property
def document_name(self) -> str | None:
@@ -363,11 +376,6 @@ class JobSource(SQLModel, table=True):
nullable=False,
),
)
raw_transcription: str | None = None
ai_metadata: dict[str, JsonValue] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True))
raw_api_response: dict[str, JsonValue] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True))
error_detail: str | None = None
executed_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
job: Optional["Job"] = Relationship(back_populates="job_sources", sa_relationship_kwargs={"lazy": "raise"})
source: Optional["Source"] = Relationship(back_populates="job_sources", sa_relationship_kwargs={"lazy": "raise"})
@@ -388,7 +396,19 @@ class ExecutionAttempt(SQLModel, table=True):
job_id: UUID = Field(foreign_key="job.id", index=True)
source_id: UUID = Field(foreign_key="source.id", index=True)
attempt_number: int = Field(ge=1)
status: JobSourceStatus
status: JobSourceStatus = Field(
sa_column=Column(
# Declared identically to job_source.status. Without values_callable
# SQLAlchemy persists enum *names*, which is defect [45]: the two
# columns spelled the same status differently and never compared equal.
SAEnum(
JobSourceStatus,
values_callable=lambda enum_cls: [item.value for item in enum_cls],
native_enum=False,
),
nullable=False,
)
)
provider: str
model: str | None = None
request_manifest: dict[str, JsonValue] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True))
src/transcription/services/jobs.py
+8 -10
View File
@@ -338,10 +338,7 @@ class JobService(ServiceBase):
for job_source in job.job_sources:
if job_source.status == JobSourceStatus.TRANSCRIBED:
continue
job_source.status = JobSourceStatus.FAILED
job_source.raw_transcription = None
job_source.error_detail = "Cancelled by user"
job_source.executed_at = now
job_source.status = JobSourceStatus.CANCELLED
await self._finalize(session=_session, caller_session=session, refresh=(job,))
return job
@@ -368,20 +365,21 @@ class JobService(ServiceBase):
suggestion="Cancel processing first, then resubmit remaining sources.",
)
candidates = [job_source for job_source in job.job_sources if job_source.status == JobSourceStatus.FAILED]
# Cancelled pages are re-attemptable: before V4.7 cancel wrote FAILED,
# so resubmit already reset them. Excluding CANCELLED here would make
# cancelled work permanently unrecoverable.
resubmittable = {JobSourceStatus.FAILED, JobSourceStatus.CANCELLED}
candidates = [job_source for job_source in job.job_sources if job_source.status in resubmittable]
if not candidates:
raise JobResubmitBlockedError(
"Job has no failed sources to resubmit",
"Job has no failed or cancelled sources to resubmit",
category=ErrorCategory.VALIDATION,
suggestion="Only failed sources can be resubmitted.",
suggestion="Only failed or cancelled sources can be resubmitted.",
)
now = datetime.now(UTC)
for job_source in candidates:
job_source.status = JobSourceStatus.PENDING
job_source.raw_transcription = None
job_source.error_detail = None
job_source.executed_at = now
job.status = JobStatus.QUEUED
job.date_updated = now
src/transcription/services/sources.py
+9 -17
View File
@@ -350,7 +350,11 @@ class SourceService(ServiceBase):
async with self._session_scope(session) as _session:
query = select(Source).options(
selectinload(Source.document),
selectinload(Source.job_sources),
# Both are needed by Source.latest_job_source and
# latest_error_detail: recency comes from the parent job, and
# failure detail lives on the attempt, not the junction row.
selectinload(Source.job_sources).selectinload(orm_attribute(JobSource.job)),
selectinload(Source.job_sources).selectinload(orm_attribute(JobSource.execution_attempts)),
)
if document_id is not None:
query = query.where(Source.document_id == document_id)
@@ -567,24 +571,12 @@ class SourceService(ServiceBase):
select(JobSource).where(JobSource.job_id == job_id).where(JobSource.source_id == source_id)
)
job_source = existing_job_source.first()
outcome = JobSourceStatus.TRANSCRIBED if text is not None else JobSourceStatus.FAILED
if job_source is None:
job_source = JobSource(
job_id=job_id,
source_id=source_id,
status=JobSourceStatus.TRANSCRIBED if text is not None else JobSourceStatus.FAILED,
raw_transcription=text,
ai_metadata=metadata_payload,
raw_api_response=raw_response_payload,
error_detail=error_detail,
)
job_source = JobSource(job_id=job_id, source_id=source_id, status=outcome)
_session.add(job_source)
else:
job_source.raw_transcription = text
job_source.ai_metadata = metadata_payload
job_source.raw_api_response = raw_response_payload
job_source.error_detail = error_detail
job_source.status = JobSourceStatus.TRANSCRIBED if text is not None else JobSourceStatus.FAILED
job_source.executed_at = datetime.now(UTC)
job_source.status = outcome
finish_time = finished_at or datetime.now(UTC)
start_time = started_at or finish_time
@@ -605,7 +597,7 @@ class SourceService(ServiceBase):
job_id=job_id,
source_id=source_id,
attempt_number=(attempt_number or 0) + 1,
status=JobSourceStatus.TRANSCRIBED if text is not None else JobSourceStatus.FAILED,
status=outcome,
provider=provider or job.provider or self.settings.provider.value,
model=model or job.model,
request_manifest=manifest_payload,
src/transcription/services/workflows.py
+7 -1
View File
@@ -411,7 +411,13 @@ async def process_next_queued_job(
def _resolve_job_sources(job: Job) -> list[Source]:
"""Resolve non-transcribed linked sources for a job in deterministic page order."""
"""Resolve pending linked sources for a job in deterministic page order.
A page is work if it has not already succeeded. CANCELLED is included
deliberately: resubmit resets cancelled pages to PENDING, so they are
re-attemptable, and a cancelled page that somehow reaches a running job is
unfinished work rather than a terminal outcome.
"""
if not job.job_sources:
return []
src/transcription/ui/pages/jobs_page.py
+12 -6
View File
@@ -284,9 +284,10 @@ def register_page() -> None: # noqa: PLR0915
with archival_card(extra_classes="gap-2"):
ui.label(f"Job ID: {job.id}").classes("text-sm font-semibold font-mono ui-text-primary")
metadata_row("Current Status:", job.status.value)
ui.label("Cancel stops processing and marks remaining non-transcribed sources as failed.").classes(
"text-xs ui-text-muted"
)
ui.label(
"Cancel stops processing and marks remaining non-transcribed sources as cancelled. "
"Cancelled sources can be resubmitted."
).classes("text-xs ui-text-muted")
async def submit_cancel() -> None:
try:
@@ -327,7 +328,11 @@ def register_page() -> None: # noqa: PLR0915
render_record_not_found("Job")
return
failed_count = sum(1 for js in job.job_sources if js.status == JobSourceStatus.FAILED)
resubmittable_count = sum(
1
for js in job.job_sources
if js.status in {JobSourceStatus.FAILED, JobSourceStatus.CANCELLED}
)
with ui.column().classes("w-full max-w-xl mx-auto p-4 gap-4"):
page_header("Resubmit Job")
@@ -335,9 +340,10 @@ def register_page() -> None: # noqa: PLR0915
with archival_card(extra_classes="gap-2"):
ui.label(f"Job ID: {job.id}").classes("text-sm font-semibold font-mono ui-text-primary")
metadata_row("Current Status:", job.status.value)
metadata_row("Failed Sources:", str(failed_count))
metadata_row("Resubmittable Sources:", str(resubmittable_count))
ui.label(
"Resubmit queues only failed linked sources. Prior execution evidence remains preserved."
"Resubmit queues failed and cancelled linked sources. "
"Prior execution evidence remains preserved."
).classes("text-xs ui-text-muted")
async def submit_resubmit() -> None:
src/transcription/ui/pages/sources_page.py
+21 -32
View File
@@ -12,6 +12,7 @@ from nicegui import ui
from transcription.config import Settings
from transcription.db.models import ExecutionAttempt
from transcription.db.models import JobSource
from transcription.db.models import JobSourceStatus
from transcription.db.models import Source
from transcription.services.sources import LatestExecutionAttempt
from transcription.services.sources import SourceDeleteBlockedError
@@ -120,7 +121,7 @@ def register_page() -> None: # noqa: PLR0915
try:
source = await sources_service.read_source_detail(parsed_source_id)
navigation = await sources_service.read_source_navigation(parsed_source_id)
latest_job_source = _latest_job_source(source)
latest_job_source = source.latest_job_source
latest_attempt = (
await sources_service.read_latest_execution_attempt(job_source_id=latest_job_source.id)
if latest_job_source is not None
@@ -134,7 +135,7 @@ def register_page() -> None: # noqa: PLR0915
show_error(exc, title="Load failed", operation="sources.read")
return
original_transcription = _resolve_original_transcription(source=source, latest_job_source=latest_job_source)
original_transcription = _resolve_original_transcription(source=source, latest_attempt=latest_attempt)
with ui.column().classes("w-full max-w-[1800px] mx-auto p-4 gap-4"):
with section_header_row():
@@ -336,7 +337,10 @@ def _render_source_job_metadata_zone(
archival_badge(status)
metadata_row("Job ID:", str(latest_job_source.job_id))
metadata_row("Executed:", latest_job_source.executed_at.isoformat())
metadata_row(
"Executed:",
latest_attempt.attempt.finished_at.isoformat() if latest_attempt is not None else "not yet executed",
)
metadata_row(
"Provider:",
latest_job_source.job.provider if latest_job_source.job and latest_job_source.job.provider else "unknown",
@@ -352,30 +356,18 @@ def _render_source_job_metadata_zone(
else "unknown",
)
if latest_job_source.error_detail:
if latest_attempt is not None and latest_attempt.attempt.error_detail:
with ui.column().classes("w-full mt-2"):
ui.label("Failure Detail:").classes("ui-text-muted text-xs mb-1")
ui.label(latest_job_source.error_detail).classes("p-2 ui-note-box text-xs")
ui.label(latest_attempt.attempt.error_detail).classes("p-2 ui-note-box text-xs")
_render_provider_evidence(
latest_job_source,
latest_attempt=latest_attempt,
)
_render_provider_evidence(latest_attempt=latest_attempt)
def _render_provider_evidence(
job_source: JobSource,
*,
latest_attempt: LatestExecutionAttempt | None,
) -> None:
def _render_provider_evidence(*, latest_attempt: LatestExecutionAttempt | None) -> None:
ui.label("Provider Evidence").classes("text-xs font-semibold ui-text-primary mt-3")
if latest_attempt is None:
render_empty_state("Exact transport evidence was not captured for this historical execution.", italic=True)
_render_json_evidence("Normalized Metadata (AI Metadata)", job_source.ai_metadata)
_render_json_evidence(
"OpenRouter SDK Response Snapshot (Raw API Response compatibility field)",
job_source.raw_api_response,
)
return
attempt = latest_attempt.attempt
@@ -516,10 +508,13 @@ def _render_source_transcription_zone(
icon="refresh",
).props("flat")
if latest_job_source is not None and latest_job_source.status.value == "failed":
ui.label("Source has a failed job execution. Save a human revision to preserve corrected text.").classes(
"text-xs ui-text-muted italic"
)
if latest_job_source is not None and latest_job_source.status in {
JobSourceStatus.FAILED,
JobSourceStatus.CANCELLED,
}:
ui.label(
"Source has an unfinished job execution. Save a human revision to preserve corrected text."
).classes("text-xs ui-text-muted italic")
def _render_machine_candidates(
@@ -651,13 +646,7 @@ def _reset_revision_text(revision_input: ui.textarea, source: Source, original_t
revision_input.value = fallback_text
def _latest_job_source(source: Source) -> JobSource | None:
if not source.job_sources:
return None
return max(source.job_sources, key=lambda item: item.executed_at)
def _resolve_original_transcription(*, source: Source, latest_job_source: JobSource | None) -> str | None:
if source.raw_transcription is None and latest_job_source is not None:
return latest_job_source.raw_transcription
def _resolve_original_transcription(*, source: Source, latest_attempt: LatestExecutionAttempt | None) -> str | None:
if source.raw_transcription is None and latest_attempt is not None:
return latest_attempt.attempt.raw_transcription
return source.raw_transcription