generated from john/python-template
zoltan57andCopilot App
6a3ee26733
Fixes the three ui.instructions.md violations recorded as [HIGH-07] and extracts the page-level duplication catalogued in review section 4. Boundary violations - jobs_page no longer imports session_scope or manages a transaction. store.create_document_job and store.create_job_for_document accept an optional session_factory and open their own session scope when the caller supplies neither a session nor a factory. - sources_page no longer calls sqlalchemy.inspect. SourceService .read_latest_execution_attempt now returns a LatestExecutionAttempt read model carrying a plain transport_body_deferred flag, so ORM loader state stays inside the service. Rendered output is unchanged. - Deletes ui/components/document_panzoom.py, its export, and its CSS. The component was exported but used by no page. Pan-zoom is planned for a clean reintroduction in V4.7 alongside the other photo/image work. Extracted duplication - ui/components/media_urls.py: pure upload-URL resolution taking upload_dir and base_url, replacing two identical ~60-line copies in sources_page and people_page. - ui/components/guards.py: parse-then-render-terminal-message, replacing 28 hand-written guard labels across five pages. - ui/components/confirm_delete.py: the blocked-dependency notice and the delete/cancel action row, from four delete pages. - ui/components/upload_panel.py: the auto-uploading file picker, from three pages. Source accept lists now derive from services.source_media .SOURCE_EXTENSIONS instead of being hard-coded. - ui/components/table/registry.py: the two hand-rolled label-registry tables on the settings page now go through build_table, which gained selection and rows_per_page options. - ui/components/formatters.py gains parse_uuid and parse_iso_date, replacing five and two private copies. - ui/runtime.py owns resolve_runtime_settings, replacing three copies and removing get_settings from every page module. [LOW-05] - Upload handlers are annotated with events.UploadEventArguments. - The Document and Person form builders return DocumentFormFields and PersonFormFields dataclasses instead of dict[str, Any]. Verification - tests/test_ui_boundaries.py asserts no page imports a session scope, a session factory, get_settings, sqlalchemy, or sqlmodel, and that no component imports request or application state. - 275 passed, 4 skipped. ruff check clean. Findings: HIGH-07, LOW-05 Co-authored-by: Copilot App <[email protected]>
698 lines
26 KiB
Python
698 lines
26 KiB
Python
import asyncio
|
|
import inspect
|
|
import logging
|
|
from dataclasses import dataclass
|
|
from datetime import UTC
|
|
from datetime import datetime
|
|
from uuid import UUID
|
|
|
|
from sqlmodel.ext.asyncio.session import AsyncSession
|
|
|
|
from ..config import Settings
|
|
from ..config import get_settings
|
|
from ..db.models import Document
|
|
from ..db.models import Job
|
|
from ..db.models import JobPurpose
|
|
from ..db.models import JobSource
|
|
from ..db.models import JobSourceStatus
|
|
from ..db.models import JobStatus
|
|
from ..db.models import Source
|
|
from ..db.session import transaction_scope
|
|
from ..errors import AppError
|
|
from ..errors import ErrorCategory
|
|
from ..errors import classify_unexpected_error
|
|
from ..errors import format_error_detail
|
|
from ..providers import ProviderError
|
|
from ..providers import RequestManifest
|
|
from ..providers import SourceEvidenceReference
|
|
from ..providers import TranscriptionProvider
|
|
from ..providers import TranscriptionResult
|
|
from ..providers import TransportEvidence
|
|
from . import ServiceBundle
|
|
from .documents import DocumentService
|
|
from .people import DocumentPersonInput
|
|
from .people import PeopleService
|
|
from .quality import QUALITY_ANALYSIS_PRODUCER
|
|
from .quality import QUALITY_ANALYSIS_PRODUCER_VERSION
|
|
from .quality import QUALITY_ANALYSIS_SCHEMA
|
|
from .quality import QUALITY_ANALYSIS_VERSION
|
|
from .quality import analyze_transcription_quality
|
|
from .quality import quality_warning_payload
|
|
from .sources import PromptExecution
|
|
from .sources import build_prompt_execution
|
|
from .sources import hash_prompt_text
|
|
from .sources import transcribe_document_image
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def create_document_with_people(
|
|
*,
|
|
document: Document,
|
|
links: list[DocumentPersonInput],
|
|
documents: DocumentService,
|
|
people: PeopleService,
|
|
) -> Document:
|
|
"""Create a Document and its complete Linked People set atomically."""
|
|
async with transaction_scope(session_factory=documents.session_factory) as session:
|
|
created = await documents.create_document(document, session=session)
|
|
await people.sync_document_people(document_id=created.id, links=links, session=session)
|
|
return created
|
|
|
|
|
|
async def update_document_with_people(
|
|
*,
|
|
document: Document,
|
|
links: list[DocumentPersonInput],
|
|
documents: DocumentService,
|
|
people: PeopleService,
|
|
) -> Document:
|
|
"""Update a Document and its complete Linked People set atomically."""
|
|
async with transaction_scope(session_factory=documents.session_factory) as session:
|
|
updated = await documents.update_document(document, session=session)
|
|
await people.sync_document_people(document_id=updated.id, links=links, session=session)
|
|
return updated
|
|
|
|
|
|
async def create_source_retranscription_job(
|
|
*,
|
|
source_id,
|
|
model: str,
|
|
services: ServiceBundle,
|
|
settings: Settings | None = None,
|
|
) -> Job:
|
|
"""Create one immutable queued retranscription Job for an existing Source."""
|
|
runtime_settings = settings or get_settings()
|
|
if model not in runtime_settings.provider_models:
|
|
raise AppError(
|
|
f"Model is not configured for retranscription: {model}",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Select one of the configured provider models.",
|
|
)
|
|
prompt = build_prompt_execution(settings=runtime_settings)
|
|
async with transaction_scope(session_factory=services.jobs.session_factory) as session:
|
|
source = await services.sources.read_source(source_id, session=session)
|
|
job = await services.jobs.create_job(
|
|
Job(
|
|
document_id=source.document_id,
|
|
purpose=JobPurpose.RETRANSCRIPTION,
|
|
provider=runtime_settings.provider.value,
|
|
model=model,
|
|
prompt_name=prompt.prompt_name,
|
|
prompt_hash=prompt.prompt_hash,
|
|
system_prompt=prompt.system_prompt,
|
|
user_prompt=prompt.user_prompt,
|
|
temperature=prompt.temperature,
|
|
top_p=prompt.top_p,
|
|
),
|
|
session=session,
|
|
)
|
|
await services.sources.create_job_source(
|
|
JobSource(job_id=job.id, source_id=source.id),
|
|
session=session,
|
|
)
|
|
return job
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _SuccessfulPage:
|
|
source: Source
|
|
result: TranscriptionResult
|
|
started_at: datetime
|
|
finished_at: datetime
|
|
duration_ms: int
|
|
model_input_artifact_id: UUID | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _FailedPage:
|
|
source: Source
|
|
error: AppError
|
|
started_at: datetime
|
|
finished_at: datetime
|
|
duration_ms: int
|
|
request_manifest: RequestManifest | None = None
|
|
transport_evidence: TransportEvidence | None = None
|
|
failure_phase: str | None = None
|
|
sdk_response_snapshot: dict | None = None
|
|
normalized_metadata: dict | None = None
|
|
provider: str | None = None
|
|
model: str | None = None
|
|
model_input_artifact_id: UUID | None = None
|
|
|
|
|
|
async def advance_job(
|
|
job: Job,
|
|
services: ServiceBundle,
|
|
settings: Settings | None = None,
|
|
session: AsyncSession | None = None,
|
|
) -> Job | None:
|
|
"""Advance a single job by lifecycle status."""
|
|
settings = settings or get_settings()
|
|
current_status = _coerce_job_status(job.status)
|
|
match current_status:
|
|
case JobStatus.QUEUED:
|
|
return await process_queued_job(job=job, services=services, settings=settings, session=session)
|
|
case JobStatus.PROCESSING:
|
|
# Recover mid-flight jobs by continuing the queued processing path.
|
|
return await process_queued_job(job=job, services=services, settings=settings, session=session)
|
|
case JobStatus.FAILED:
|
|
if job.retry_count < settings.worker_max_retries:
|
|
return await services.jobs.update_job_state(
|
|
job_id=job.id,
|
|
status=JobStatus.QUEUED,
|
|
retry_count_increment=1,
|
|
session=session,
|
|
)
|
|
else:
|
|
logger.error(f"Job {job.id} has failed and reached max retries.")
|
|
return
|
|
case _:
|
|
return
|
|
|
|
|
|
async def process_queued_job( # noqa: PLR0915
|
|
*,
|
|
job: Job,
|
|
services: ServiceBundle,
|
|
settings: Settings | None = None,
|
|
session: AsyncSession | None = None,
|
|
) -> Job | None:
|
|
"""Process one complete transcription attempt for a queued job."""
|
|
runtime_settings = settings or get_settings()
|
|
current_status = _coerce_job_status(job.status)
|
|
if current_status not in {JobStatus.QUEUED, JobStatus.PROCESSING}:
|
|
logger.warning(f"Job {job.id} is not queued. Current status: {job.status}")
|
|
return
|
|
|
|
# Transaction A: claim job for processing. Reached only when a caller hands us a
|
|
# still-QUEUED job directly; the worker path already claimed it atomically in
|
|
# JobService.claim_next_queued_job.
|
|
if current_status == JobStatus.QUEUED:
|
|
if session is None:
|
|
job = await services.jobs.mark_job_status(job.id, JobStatus.PROCESSING)
|
|
else:
|
|
# Commit the PROCESSING transition before transcription starts so the claim
|
|
# is durable and visible to any other worker before the long provider call.
|
|
job = await services.jobs.mark_job_status(job.id, JobStatus.PROCESSING, session=session)
|
|
await session.commit()
|
|
|
|
source_job = await services.jobs.read_job(job_id=job.id, session=session)
|
|
sources = _resolve_job_sources(source_job)
|
|
if not sources and not source_job.job_sources:
|
|
candidate_sources = await services.sources.list_sources(document_id=job.document_id, session=session)
|
|
sources = sorted(candidate_sources, key=lambda item: (item.page_number, item.upload_name.casefold()))
|
|
|
|
if not sources:
|
|
return await services.jobs.mark_job_status(job.id, JobStatus.TRANSCRIBED, session=session)
|
|
|
|
successful_pages: list[_SuccessfulPage] = []
|
|
failed_pages: list[_FailedPage] = []
|
|
externally_stopped = False
|
|
|
|
prompt_execution = _resolve_job_prompt_execution(source_job=source_job, settings=runtime_settings)
|
|
|
|
for source in sources:
|
|
if await _job_no_longer_processing(job_id=job.id, services=services, session=session):
|
|
externally_stopped = True
|
|
break
|
|
|
|
started_at = datetime.now(UTC)
|
|
monotonic_started_at = asyncio.get_running_loop().time()
|
|
result: TranscriptionResult | None = None
|
|
provider_input = None
|
|
page_outcome: _SuccessfulPage | _FailedPage
|
|
try:
|
|
provider_input = await services.sources.resolve_provider_input(source, session=session)
|
|
if session is not None:
|
|
await session.commit()
|
|
source_reference = SourceEvidenceReference(
|
|
source_id=source.id,
|
|
digest_sha256=provider_input.digest_sha256,
|
|
byte_size=provider_input.byte_size,
|
|
media_type=provider_input.media_type,
|
|
page_number=source.page_number,
|
|
width=provider_input.width,
|
|
height=provider_input.height,
|
|
derivative_id=provider_input.derivative_id,
|
|
transformation=provider_input.transformation,
|
|
)
|
|
result = await asyncio.wait_for(
|
|
_call_transcriber(
|
|
input_path=provider_input.path,
|
|
prompt_execution=prompt_execution,
|
|
settings=runtime_settings,
|
|
provider=services.sources.provider,
|
|
source_reference=source_reference,
|
|
requested_model=source_job.model,
|
|
),
|
|
timeout=runtime_settings.worker_provider_timeout_seconds,
|
|
)
|
|
elapsed_seconds = asyncio.get_running_loop().time() - monotonic_started_at
|
|
logger.info(
|
|
"Provider response diagnostics operation=worker.provider_response "
|
|
"job_id=%s document_id=%s source_id=%s provider=%s model=%s "
|
|
"finish_reason=%s usage_input_tokens=%s usage_output_tokens=%s usage_total_tokens=%s "
|
|
"latency_seconds=%.3f text_chars=%s text_lines=%s",
|
|
job.id,
|
|
job.document_id,
|
|
source.id,
|
|
result.provider,
|
|
result.model,
|
|
result.finish_reason or "unknown",
|
|
result.usage_input_tokens,
|
|
result.usage_output_tokens,
|
|
result.usage_total_tokens,
|
|
elapsed_seconds,
|
|
len(result.text),
|
|
_line_count(result.text),
|
|
)
|
|
|
|
_validate_transcription_quality(result=result, settings=runtime_settings)
|
|
finished_at = datetime.now(UTC)
|
|
page_outcome = _SuccessfulPage(
|
|
source=source,
|
|
result=result,
|
|
started_at=started_at,
|
|
finished_at=finished_at,
|
|
duration_ms=max(0, int(elapsed_seconds * 1000)),
|
|
model_input_artifact_id=provider_input.derivative_id,
|
|
)
|
|
successful_pages.append(page_outcome)
|
|
except TimeoutError:
|
|
error = AppError(
|
|
f"Provider call timed out after {runtime_settings.worker_provider_timeout_seconds:.1f}s",
|
|
category=ErrorCategory.EXTERNAL_PROVIDER,
|
|
suggestion="Retry the job. If this repeats, verify provider latency and request payload size.",
|
|
retriable=True,
|
|
)
|
|
finished_at = datetime.now(UTC)
|
|
page_outcome = _FailedPage(
|
|
source=source,
|
|
error=error,
|
|
started_at=started_at,
|
|
finished_at=finished_at,
|
|
duration_ms=max(
|
|
0,
|
|
int((asyncio.get_running_loop().time() - monotonic_started_at) * 1000),
|
|
),
|
|
request_manifest=services.sources.provider.current_request_manifest,
|
|
transport_evidence=services.sources.provider.current_transport_evidence,
|
|
failure_phase="local_timeout",
|
|
model_input_artifact_id=(
|
|
provider_input.derivative_id if provider_input is not None else None
|
|
),
|
|
)
|
|
failed_pages.append(page_outcome)
|
|
logger.error(
|
|
"Source failed operation=worker.process_job job_id=%s document_id=%s "
|
|
"source_id=%s error_id=%s category=%s",
|
|
job.id,
|
|
job.document_id,
|
|
source.id,
|
|
error.error_id,
|
|
error.category.value,
|
|
)
|
|
except Exception as exc: # noqa: BLE001
|
|
match exc:
|
|
case AppError() as error:
|
|
pass
|
|
case _:
|
|
error = classify_unexpected_error(exc, operation="worker.process_job")
|
|
|
|
finished_at = datetime.now(UTC)
|
|
provider_error = _find_provider_error(exc)
|
|
page_outcome = _FailedPage(
|
|
source=source,
|
|
error=error,
|
|
started_at=started_at,
|
|
finished_at=finished_at,
|
|
duration_ms=max(
|
|
0,
|
|
int((asyncio.get_running_loop().time() - monotonic_started_at) * 1000),
|
|
),
|
|
request_manifest=(
|
|
result.request_manifest
|
|
if result is not None
|
|
else provider_error.request_manifest
|
|
if provider_error is not None
|
|
else None
|
|
),
|
|
transport_evidence=(
|
|
result.transport_evidence
|
|
if result is not None
|
|
else provider_error.transport_evidence
|
|
if provider_error is not None
|
|
else None
|
|
),
|
|
failure_phase=(
|
|
"transcription_quality"
|
|
if result is not None
|
|
else provider_error.failure_phase
|
|
if provider_error is not None
|
|
else "application"
|
|
),
|
|
sdk_response_snapshot=result.raw_api_response if result is not None else None,
|
|
normalized_metadata=result.metadata_payload() if result is not None else None,
|
|
provider=result.provider if result is not None else None,
|
|
model=result.model if result is not None else None,
|
|
model_input_artifact_id=(
|
|
provider_input.derivative_id if provider_input is not None else None
|
|
),
|
|
)
|
|
failed_pages.append(page_outcome)
|
|
logger.error(
|
|
"Source failed operation=worker.process_job job_id=%s document_id=%s "
|
|
"source_id=%s error_id=%s category=%s",
|
|
job.id,
|
|
job.document_id,
|
|
source.id,
|
|
error.error_id,
|
|
error.category.value,
|
|
)
|
|
|
|
await _persist_page_outcome_durably(
|
|
job=job,
|
|
services=services,
|
|
page=page_outcome,
|
|
session=session,
|
|
)
|
|
|
|
if await _job_no_longer_processing(job_id=job.id, services=services, session=session):
|
|
externally_stopped = True
|
|
break
|
|
|
|
terminal_status = JobStatus.TRANSCRIBED
|
|
if externally_stopped:
|
|
terminal_status = JobStatus.FAILED
|
|
elif failed_pages and successful_pages:
|
|
terminal_status = JobStatus.PARTIAL_SUCCESS
|
|
elif failed_pages and not successful_pages:
|
|
terminal_status = JobStatus.FAILED
|
|
|
|
updated_job = await _finalize_batch_outcome(
|
|
job=job,
|
|
services=services,
|
|
status=terminal_status,
|
|
session=session,
|
|
)
|
|
|
|
logger.info(
|
|
"Job finished operation=worker.process_job job_id=%s document_id=%s status=%s success_pages=%s failed_pages=%s",
|
|
updated_job.id,
|
|
updated_job.document_id,
|
|
updated_job.status.value,
|
|
len(successful_pages),
|
|
len(failed_pages),
|
|
)
|
|
return updated_job
|
|
|
|
|
|
async def process_next_queued_job(
|
|
*,
|
|
services: ServiceBundle,
|
|
settings: Settings | None = None,
|
|
session: AsyncSession | None = None,
|
|
) -> bool:
|
|
"""Process the next queued job if one exists."""
|
|
job = await services.jobs.claim_next_queued_job(session=session)
|
|
|
|
if job is None:
|
|
return False
|
|
|
|
# The claim must be durable before the provider call starts, otherwise another
|
|
# worker could observe the job as still QUEUED and process it a second time.
|
|
if session is not None:
|
|
await session.commit()
|
|
|
|
await advance_job(job=job, services=services, settings=settings, session=session)
|
|
return True
|
|
|
|
|
|
def _resolve_job_sources(job: Job) -> list[Source]:
|
|
"""Resolve non-transcribed linked sources for a job in deterministic page order."""
|
|
if not job.job_sources:
|
|
return []
|
|
|
|
sources = [
|
|
job_source.source
|
|
for job_source in job.job_sources
|
|
if job_source.source is not None and job_source.status != JobSourceStatus.TRANSCRIBED
|
|
]
|
|
return sorted(sources, key=lambda item: (item.page_number, item.upload_name.casefold()))
|
|
|
|
|
|
def _resolve_job_prompt_execution(*, source_job: Job, settings: Settings) -> PromptExecution:
|
|
if source_job.user_prompt and source_job.prompt_name:
|
|
return PromptExecution(
|
|
prompt_name=source_job.prompt_name,
|
|
prompt_hash=source_job.prompt_hash or hash_prompt_text(source_job.user_prompt),
|
|
system_prompt=source_job.system_prompt,
|
|
user_prompt=source_job.user_prompt,
|
|
temperature=source_job.temperature,
|
|
top_p=source_job.top_p,
|
|
)
|
|
|
|
return build_prompt_execution(settings=settings)
|
|
|
|
|
|
async def _job_no_longer_processing(
|
|
*,
|
|
job_id,
|
|
services: ServiceBundle,
|
|
session: AsyncSession | None = None,
|
|
) -> bool:
|
|
"""Return True when job status changed externally from PROCESSING."""
|
|
latest_job = await services.jobs.read_job(job_id=job_id, session=session)
|
|
return latest_job.status != JobStatus.PROCESSING
|
|
|
|
|
|
async def _finalize_batch_outcome(
|
|
*,
|
|
job: Job,
|
|
services: ServiceBundle,
|
|
status: JobStatus,
|
|
session: AsyncSession | None = None,
|
|
) -> Job:
|
|
"""Persist the terminal aggregate status after all page outcomes are durable."""
|
|
if session is None:
|
|
async with services.jobs._session_scope() as local_session:
|
|
updated_job = await services.jobs.mark_job_status(job.id, status, session=local_session)
|
|
await local_session.commit()
|
|
return updated_job
|
|
|
|
updated_job = await services.jobs.mark_job_status(job.id, status, session=session)
|
|
await session.commit()
|
|
return updated_job
|
|
|
|
|
|
async def _persist_page_outcome_durably(
|
|
*,
|
|
job: Job,
|
|
services: ServiceBundle,
|
|
page: _SuccessfulPage | _FailedPage,
|
|
session: AsyncSession | None,
|
|
) -> None:
|
|
"""Commit one completed provider call before processing the next source."""
|
|
task = asyncio.create_task(_persist_page_outcome(job=job, services=services, page=page, session=session))
|
|
try:
|
|
await asyncio.shield(task)
|
|
except asyncio.CancelledError:
|
|
await task
|
|
raise
|
|
|
|
|
|
async def _persist_page_outcome(
|
|
*,
|
|
job: Job,
|
|
services: ServiceBundle,
|
|
page: _SuccessfulPage | _FailedPage,
|
|
session: AsyncSession | None,
|
|
) -> None:
|
|
if session is None:
|
|
async with services.sources._session_scope() as local_session:
|
|
await _write_page_outcome(job=job, services=services, page=page, session=local_session)
|
|
await local_session.commit()
|
|
return
|
|
|
|
await _write_page_outcome(job=job, services=services, page=page, session=session)
|
|
await session.commit()
|
|
|
|
|
|
async def _write_page_outcome(
|
|
*,
|
|
job: Job,
|
|
services: ServiceBundle,
|
|
page: _SuccessfulPage | _FailedPage,
|
|
session: AsyncSession,
|
|
) -> None:
|
|
if isinstance(page, _SuccessfulPage):
|
|
source = page.source
|
|
result = page.result
|
|
await services.sources.update_job_source_transcription(
|
|
job_id=job.id,
|
|
source_id=source.id,
|
|
text=result.text,
|
|
error_detail=None,
|
|
ai_metadata=result.metadata_payload(),
|
|
raw_api_response=result.raw_api_response,
|
|
provider=result.provider,
|
|
model=result.model,
|
|
request_manifest=result.request_manifest,
|
|
model_input_artifact_id=page.model_input_artifact_id,
|
|
transport_evidence=result.transport_evidence,
|
|
started_at=page.started_at,
|
|
finished_at=page.finished_at,
|
|
duration_ms=page.duration_ms,
|
|
session=session,
|
|
)
|
|
job_source = await services.sources.read_job_source_for_job(
|
|
job_id=job.id,
|
|
source_id=source.id,
|
|
session=session,
|
|
)
|
|
attempt = await services.sources.read_latest_execution_attempt(
|
|
job_source_id=job_source.id,
|
|
session=session,
|
|
)
|
|
if attempt is None:
|
|
raise RuntimeError("Successful transcription did not create execution evidence")
|
|
warnings = analyze_transcription_quality(result.text)
|
|
await services.sources.create_json_artifact(
|
|
source_id=source.id,
|
|
execution_attempt_id=attempt.attempt.id,
|
|
artifact_type="transcription_quality_warnings",
|
|
schema_name=QUALITY_ANALYSIS_SCHEMA,
|
|
schema_version=QUALITY_ANALYSIS_VERSION,
|
|
producer=QUALITY_ANALYSIS_PRODUCER,
|
|
producer_version=QUALITY_ANALYSIS_PRODUCER_VERSION,
|
|
payload=quality_warning_payload(warnings),
|
|
session=session,
|
|
)
|
|
return
|
|
|
|
await services.sources.update_job_source_transcription(
|
|
job_id=job.id,
|
|
source_id=page.source.id,
|
|
text=None,
|
|
error_detail=format_error_detail(page.error),
|
|
ai_metadata=page.normalized_metadata,
|
|
raw_api_response=page.sdk_response_snapshot,
|
|
provider=page.provider,
|
|
model=page.model,
|
|
request_manifest=page.request_manifest,
|
|
model_input_artifact_id=page.model_input_artifact_id,
|
|
transport_evidence=page.transport_evidence,
|
|
failure_phase=page.failure_phase,
|
|
error_category=page.error.category.value,
|
|
started_at=page.started_at,
|
|
finished_at=page.finished_at,
|
|
duration_ms=page.duration_ms,
|
|
session=session,
|
|
)
|
|
|
|
|
|
def _validate_transcription_quality(*, result: TranscriptionResult, settings: Settings) -> None:
|
|
text_chars = len(result.text)
|
|
text_lines = _line_count(result.text)
|
|
|
|
if settings.worker_fail_on_finish_reason_length and (result.finish_reason or "").lower() == "length":
|
|
raise AppError(
|
|
"Provider output appears truncated (finish_reason=length)",
|
|
category=ErrorCategory.EXTERNAL_PROVIDER,
|
|
suggestion=(
|
|
"Retry the job. If this repeats, use a faster model, reduce input complexity, "
|
|
"or increase provider output budget."
|
|
),
|
|
retriable=True,
|
|
)
|
|
|
|
if settings.worker_min_transcription_chars > 0 and text_chars < settings.worker_min_transcription_chars:
|
|
raise AppError(
|
|
(
|
|
"Transcription output below configured minimum character threshold "
|
|
f"({text_chars} < {settings.worker_min_transcription_chars})"
|
|
),
|
|
category=ErrorCategory.EXTERNAL_PROVIDER,
|
|
suggestion=(
|
|
"Retry the job. If this repeats, switch model or raise minimum thresholds based on document type."
|
|
),
|
|
retriable=True,
|
|
)
|
|
|
|
if settings.worker_min_transcription_lines > 0 and text_lines < settings.worker_min_transcription_lines:
|
|
raise AppError(
|
|
(
|
|
"Transcription output below configured minimum line threshold "
|
|
f"({text_lines} < {settings.worker_min_transcription_lines})"
|
|
),
|
|
category=ErrorCategory.EXTERNAL_PROVIDER,
|
|
suggestion=(
|
|
"Retry the job. If this repeats, switch model or raise minimum thresholds based on document type."
|
|
),
|
|
retriable=True,
|
|
)
|
|
|
|
|
|
def _line_count(text: str) -> int:
|
|
stripped = text.strip()
|
|
if not stripped:
|
|
return 0
|
|
return sum(1 for line in stripped.splitlines() if line.strip())
|
|
|
|
|
|
def _coerce_job_status(value: object) -> JobStatus | None:
|
|
if isinstance(value, JobStatus):
|
|
return value
|
|
|
|
if isinstance(value, str):
|
|
lowered = value.strip().lower()
|
|
for member in JobStatus:
|
|
if lowered in {member.value.lower(), member.name.lower()}:
|
|
return member
|
|
|
|
return None
|
|
|
|
|
|
def _find_provider_error(exc: BaseException) -> ProviderError | None:
|
|
"""Find provider evidence carried through application error translation."""
|
|
current: BaseException | None = exc
|
|
while current is not None:
|
|
if isinstance(current, ProviderError):
|
|
return current
|
|
current = current.__cause__ or current.__context__
|
|
return None
|
|
|
|
|
|
async def _call_transcriber(
|
|
*,
|
|
input_path,
|
|
prompt_execution: PromptExecution,
|
|
settings: Settings,
|
|
provider: TranscriptionProvider,
|
|
source_reference: SourceEvidenceReference,
|
|
requested_model: str | None,
|
|
) -> TranscriptionResult:
|
|
"""Call the current transcriber while supporting legacy injected test doubles."""
|
|
if "source_reference" in inspect.signature(transcribe_document_image).parameters:
|
|
return await transcribe_document_image(
|
|
input_path,
|
|
prompt_name=prompt_execution.prompt_name,
|
|
prompt_text=prompt_execution.user_prompt,
|
|
temperature=prompt_execution.temperature,
|
|
top_p=prompt_execution.top_p,
|
|
settings=settings,
|
|
provider=provider,
|
|
source_reference=source_reference,
|
|
requested_model=requested_model,
|
|
)
|
|
return await transcribe_document_image(
|
|
input_path,
|
|
prompt_name=prompt_execution.prompt_name,
|
|
prompt_text=prompt_execution.user_prompt,
|
|
temperature=prompt_execution.temperature,
|
|
top_p=prompt_execution.top_p,
|
|
settings=settings,
|
|
provider=provider,
|
|
)
|