generated from john/python-template
1163 lines
49 KiB
Python
1163 lines
49 KiB
Python
"""Source persistence, media policy, revisions, and transcription execution."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
from collections.abc import Sequence
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass
|
|
from datetime import UTC
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from uuid import UUID
|
|
from uuid import uuid4
|
|
|
|
from pydantic import BaseModel
|
|
from pydantic import ConfigDict
|
|
from pydantic import Field
|
|
from pydantic import JsonValue
|
|
from pydantic import TypeAdapter
|
|
from pydantic import ValidationError
|
|
from sqlalchemy import func
|
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
|
from sqlalchemy.orm import defer
|
|
from sqlalchemy.orm import selectinload
|
|
from sqlmodel import select
|
|
from sqlmodel.ext.asyncio.session import AsyncSession
|
|
|
|
from transcription.config import Settings
|
|
from transcription.config import get_settings
|
|
from transcription.db.models import ExecutionAttempt
|
|
from transcription.db.models import Job
|
|
from transcription.db.models import JobSource
|
|
from transcription.db.models import JobSourceStatus
|
|
from transcription.db.models import ProcessingArtifact
|
|
from transcription.db.models import Source
|
|
from transcription.errors import AppError
|
|
from transcription.errors import ErrorCategory
|
|
from transcription.providers import ProviderAuthError
|
|
from transcription.providers import ProviderError
|
|
from transcription.providers import ProviderResponseError
|
|
from transcription.providers import RequestManifest
|
|
from transcription.providers import SourceEvidenceReference
|
|
from transcription.providers import TranscriptionMetadata
|
|
from transcription.providers import TranscriptionProvider
|
|
from transcription.providers import TranscriptionResult
|
|
from transcription.providers import TransportEvidence
|
|
from transcription.providers import get_transcription_provider
|
|
from transcription.providers.evidence import canonical_json_bytes
|
|
|
|
from .base import ServiceBase
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DEFAULT_PROMPT_FILE = "transcribe_document.md"
|
|
SOURCE_MIME_TYPES = {
|
|
".jpg": "image/jpeg",
|
|
".jpeg": "image/jpeg",
|
|
".png": "image/png",
|
|
".tif": "image/tiff",
|
|
".tiff": "image/tiff",
|
|
".pdf": "application/pdf",
|
|
}
|
|
SOURCE_EXTENSIONS = frozenset(SOURCE_MIME_TYPES)
|
|
JSON_OBJECT_ADAPTER = TypeAdapter(dict[str, JsonValue])
|
|
|
|
|
|
class PromptExecution(BaseModel):
|
|
"""Resolved prompt inputs captured for one page execution."""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True)
|
|
|
|
prompt_name: str = Field(min_length=1, pattern=r"^[^/\\]+$")
|
|
prompt_hash: str = Field(pattern=r"^[0-9a-f]{64}$")
|
|
system_prompt: str | None
|
|
user_prompt: str = Field(min_length=1)
|
|
temperature: float | None = Field(ge=0.0, le=2.0)
|
|
top_p: float | None = Field(ge=0.0, le=1.0)
|
|
|
|
|
|
class PromptLoadError(AppError):
|
|
"""Raised when prompt artifacts cannot be loaded safely."""
|
|
|
|
|
|
class TranscriptionError(AppError):
|
|
"""Raised when transcription execution fails."""
|
|
|
|
|
|
class TranscriptionNotFoundError(TranscriptionError):
|
|
"""Raised when a transcription-related resource is not found."""
|
|
|
|
|
|
class SourceDeleteBlockedError(TranscriptionError):
|
|
"""Raised when source deletion is blocked by dependency policy."""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class SourceNavigation:
|
|
"""Adjacent Source identifiers within one ordered Document."""
|
|
|
|
previous_id: UUID | None
|
|
next_id: UUID | None
|
|
|
|
|
|
class SourceService(ServiceBase):
|
|
"""Manage source records, media payloads, revisions, and page execution output."""
|
|
|
|
def __init__(
|
|
self,
|
|
session_factory: async_sessionmaker[AsyncSession] | None = None,
|
|
settings: Settings | None = None,
|
|
):
|
|
super().__init__(session_factory=session_factory, settings=settings)
|
|
self._provider: TranscriptionProvider | None = None
|
|
|
|
@property
|
|
def provider(self) -> TranscriptionProvider:
|
|
if self._provider is None:
|
|
self._provider = get_transcription_provider(settings=self.settings)
|
|
return self._provider
|
|
|
|
async def aclose(self) -> None:
|
|
"""Close provider-owned network resources when they were initialized."""
|
|
if self._provider is None:
|
|
return
|
|
close = getattr(self._provider, "aclose", None)
|
|
if close is not None:
|
|
await close()
|
|
self._provider = None
|
|
|
|
async def create_source(self, source: Source, *, session: AsyncSession | None = None) -> Source:
|
|
"""Create a new source page record in the database."""
|
|
async with self._session_scope(session) as _session:
|
|
_session.add(source)
|
|
await self._finalize(session=_session, caller_session=session, refresh=(source,))
|
|
return source
|
|
|
|
async def read_source(self, source_id: UUID, *, session: AsyncSession | None = None) -> Source:
|
|
"""Read an existing source page record."""
|
|
async with self._session_scope(session) as _session:
|
|
source = await _session.get(Source, source_id)
|
|
if source is None:
|
|
raise TranscriptionNotFoundError(
|
|
f"Source with id {source_id} not found",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Verify the source id and retry.",
|
|
)
|
|
return source
|
|
|
|
async def read_source_detail(self, source_id: UUID, *, session: AsyncSession | None = None) -> Source:
|
|
"""Read a source page record with job-source context for UI detail rendering."""
|
|
async with self._session_scope(session) as _session:
|
|
query = (
|
|
select(Source)
|
|
.options(
|
|
selectinload(Source.document), # pyright: ignore[reportArgumentType]
|
|
selectinload(Source.job_sources).selectinload(JobSource.job), # pyright: ignore[reportArgumentType]
|
|
)
|
|
.where(Source.id == source_id)
|
|
.execution_options(populate_existing=True)
|
|
)
|
|
source = (await _session.exec(query)).first()
|
|
|
|
if source is None:
|
|
raise TranscriptionNotFoundError(
|
|
f"Source with id {source_id} not found",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Verify the source id and retry.",
|
|
)
|
|
return source
|
|
|
|
async def read_latest_execution_attempt(
|
|
self,
|
|
*,
|
|
job_source_id: UUID,
|
|
session: AsyncSession | None = None,
|
|
) -> ExecutionAttempt | None:
|
|
"""Read only the latest immutable attempt for one compatibility projection."""
|
|
async with self._session_scope(session) as _session:
|
|
query = (
|
|
select(ExecutionAttempt)
|
|
.options(defer(ExecutionAttempt.transport_body)) # pyright: ignore[reportArgumentType]
|
|
.where(ExecutionAttempt.job_source_id == job_source_id)
|
|
.order_by(
|
|
ExecutionAttempt.attempt_number.desc(), # pyright: ignore[reportAttributeAccessIssue]
|
|
ExecutionAttempt.id.desc(), # pyright: ignore[reportAttributeAccessIssue]
|
|
)
|
|
.limit(1)
|
|
)
|
|
return (await _session.exec(query)).first()
|
|
|
|
async def read_source_navigation(
|
|
self,
|
|
source_id: UUID,
|
|
*,
|
|
session: AsyncSession | None = None,
|
|
) -> SourceNavigation:
|
|
"""Return adjacent Sources ordered within the current Document."""
|
|
async with self._session_scope(session) as _session:
|
|
source = await _session.get(Source, source_id)
|
|
if source is None:
|
|
raise TranscriptionNotFoundError(
|
|
f"Source with id {source_id} not found",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Verify the source id and retry.",
|
|
)
|
|
query = (
|
|
select(Source.id)
|
|
.where(Source.document_id == source.document_id)
|
|
.order_by(Source.page_number, Source.id) # pyright: ignore[reportArgumentType]
|
|
)
|
|
source_ids = list((await _session.exec(query)).all())
|
|
|
|
current_index = source_ids.index(source_id)
|
|
return SourceNavigation(
|
|
previous_id=source_ids[current_index - 1] if current_index > 0 else None,
|
|
next_id=source_ids[current_index + 1] if current_index + 1 < len(source_ids) else None,
|
|
)
|
|
|
|
async def update_source(self, source: Source, *, session: AsyncSession | None = None) -> Source:
|
|
"""Update an existing source page record."""
|
|
async with self._session_scope(session) as _session:
|
|
merged = await _session.merge(source)
|
|
await self._finalize(session=_session, caller_session=session, refresh=(merged,))
|
|
return merged
|
|
|
|
async def delete_source(self, source: Source, *, session: AsyncSession | None = None) -> None:
|
|
"""Delete a source only when it has no retained execution evidence."""
|
|
await self.delete_unlinked_source(source_id=source.id, session=session)
|
|
|
|
async def delete_unlinked_source(self, *, source_id: UUID, session: AsyncSession | None = None) -> None:
|
|
"""Delete a source only when no JobSource links exist."""
|
|
async with self._session_scope(session) as _session:
|
|
source = await _session.get(
|
|
Source,
|
|
source_id,
|
|
options=(
|
|
selectinload(Source.job_sources), # pyright: ignore[reportArgumentType]
|
|
selectinload(Source.processing_artifacts), # pyright: ignore[reportArgumentType]
|
|
),
|
|
)
|
|
if source is None:
|
|
raise TranscriptionNotFoundError(
|
|
f"Source with id {source_id} not found",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Verify the source id and retry.",
|
|
)
|
|
|
|
if source.job_sources or source.processing_artifacts:
|
|
raise SourceDeleteBlockedError(
|
|
"Source delete blocked because retained execution evidence exists",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Preserve the source or use an explicit evidence-retention workflow.",
|
|
)
|
|
|
|
source_file_path = source.file_path
|
|
await _session.delete(source)
|
|
await self._finalize(session=_session, caller_session=session)
|
|
|
|
self._delete_source_file(source_file_path=source_file_path)
|
|
|
|
async def list_sources(
|
|
self,
|
|
*,
|
|
document_id: UUID | None = None,
|
|
session: AsyncSession | None = None,
|
|
) -> Sequence[Source]:
|
|
"""List source pages, optionally filtered by document."""
|
|
async with self._session_scope(session) as _session:
|
|
query = select(Source)
|
|
if document_id is not None:
|
|
query = query.where(Source.document_id == document_id)
|
|
result = await _session.exec(query)
|
|
return result.all()
|
|
|
|
async def query_sources(
|
|
self,
|
|
*,
|
|
document_id: UUID | None = None,
|
|
page_number: int | None = None,
|
|
session: AsyncSession | None = None,
|
|
) -> Sequence[Source]:
|
|
"""Query source pages using the provided filters."""
|
|
async with self._session_scope(session) as _session:
|
|
query = select(Source)
|
|
if document_id is not None:
|
|
query = query.where(Source.document_id == document_id)
|
|
if page_number is not None:
|
|
query = query.where(Source.page_number == page_number)
|
|
result = await _session.exec(query)
|
|
return result.all()
|
|
|
|
async def list_sources_detail(
|
|
self,
|
|
*,
|
|
document_id: UUID | None = None,
|
|
job_id: UUID | None = None,
|
|
session: AsyncSession | None = None,
|
|
) -> Sequence[Source]:
|
|
"""List source pages with document/job link context for UI rendering."""
|
|
async with self._session_scope(session) as _session:
|
|
query = select(Source).options(
|
|
selectinload(Source.document),
|
|
selectinload(Source.job_sources),
|
|
)
|
|
if document_id is not None:
|
|
query = query.where(Source.document_id == document_id)
|
|
|
|
result = await _session.exec(query)
|
|
sources = list(result.all())
|
|
|
|
if job_id is not None:
|
|
sources = [
|
|
source
|
|
for source in sources
|
|
if any(job_source.job_id == job_id for job_source in source.job_sources)
|
|
]
|
|
|
|
return sources
|
|
|
|
async def create_job_source(
|
|
self,
|
|
job_source: JobSource,
|
|
*,
|
|
session: AsyncSession | None = None,
|
|
) -> JobSource:
|
|
"""Create a new job_source execution record in the database."""
|
|
async with self._session_scope(session) as _session:
|
|
_session.add(job_source)
|
|
await self._finalize(session=_session, caller_session=session, refresh=(job_source,))
|
|
return job_source
|
|
|
|
async def read_job_source(self, job_source_id: UUID, *, session: AsyncSession | None = None) -> JobSource:
|
|
"""Read an existing job_source record."""
|
|
async with self._session_scope(session) as _session:
|
|
job_source = await _session.get(
|
|
JobSource,
|
|
job_source_id,
|
|
options=(selectinload(JobSource.source),), # pyright: ignore[reportArgumentType]
|
|
)
|
|
if job_source is None:
|
|
raise TranscriptionNotFoundError(
|
|
f"JobSource with id {job_source_id} not found",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Verify the job source id and retry.",
|
|
)
|
|
return job_source
|
|
|
|
async def update_job_source(self, job_source: JobSource, *, session: AsyncSession | None = None) -> JobSource:
|
|
"""Update an existing job_source record."""
|
|
async with self._session_scope(session) as _session:
|
|
merged = await _session.merge(job_source)
|
|
await self._finalize(session=_session, caller_session=session, refresh=(merged,))
|
|
return merged
|
|
|
|
async def delete_job_source(self, job_source: JobSource, *, session: AsyncSession | None = None) -> None:
|
|
"""Delete a job_source record."""
|
|
async with self._session_scope(session) as _session:
|
|
await _session.delete(job_source)
|
|
await self._finalize(session=_session, caller_session=session)
|
|
|
|
async def delete_source_from_job_context(
|
|
self,
|
|
*,
|
|
job_id: UUID,
|
|
source_id: UUID,
|
|
session: AsyncSession | None = None,
|
|
) -> None:
|
|
"""Delete a source from an active job context with dependency guardrails.
|
|
|
|
Policy:
|
|
- Allowed when exactly one JobSource link exists and it points at ``job_id``.
|
|
- Blocked when additional JobSource links exist (history/shared dependencies).
|
|
"""
|
|
async with self._session_scope(session) as _session:
|
|
source = await _session.get(
|
|
Source,
|
|
source_id,
|
|
options=(
|
|
selectinload(Source.job_sources), # pyright: ignore[reportArgumentType]
|
|
selectinload(Source.processing_artifacts), # pyright: ignore[reportArgumentType]
|
|
),
|
|
)
|
|
if source is None:
|
|
raise TranscriptionNotFoundError(
|
|
f"Source with id {source_id} not found",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Verify the source id and retry.",
|
|
)
|
|
|
|
linked_job_sources = list(source.job_sources)
|
|
attempt_count = (
|
|
await _session.exec(
|
|
select(func.count())
|
|
.select_from(ExecutionAttempt)
|
|
.where(ExecutionAttempt.source_id == source_id)
|
|
)
|
|
).one()
|
|
if source.processing_artifacts or attempt_count:
|
|
raise SourceDeleteBlockedError(
|
|
"Source delete blocked because immutable evidence exists",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Preserve the source or use an explicit evidence-retention workflow.",
|
|
)
|
|
|
|
matching_links = [job_source for job_source in linked_job_sources if job_source.job_id == job_id]
|
|
if not matching_links:
|
|
raise TranscriptionNotFoundError(
|
|
f"Source {source_id} is not linked to job {job_id}",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Open the source from its linked job context and retry.",
|
|
)
|
|
|
|
if len(linked_job_sources) > len(matching_links):
|
|
raise SourceDeleteBlockedError(
|
|
"Source delete blocked by related job history",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Remove additional JobSource links first, then retry deletion.",
|
|
)
|
|
|
|
for job_source in matching_links:
|
|
await _session.delete(job_source)
|
|
|
|
source_file_path = source.file_path
|
|
await _session.delete(source)
|
|
await self._finalize(session=_session, caller_session=session)
|
|
|
|
self._delete_source_file(source_file_path=source_file_path)
|
|
|
|
def _delete_source_file(self, *, source_file_path: str) -> None:
|
|
"""Best-effort cleanup for source media files."""
|
|
candidate_path = Path(source_file_path)
|
|
resolved_path = candidate_path if candidate_path.is_absolute() else self.settings.upload_dir / candidate_path
|
|
|
|
if not resolved_path.exists():
|
|
return
|
|
|
|
try:
|
|
resolved_path.unlink()
|
|
logger.info("Deleted source file: %s", resolved_path)
|
|
except OSError:
|
|
logger.warning("Failed to delete source file: %s", resolved_path)
|
|
|
|
async def list_job_sources(
|
|
self,
|
|
*,
|
|
job_id: UUID | None = None,
|
|
session: AsyncSession | None = None,
|
|
) -> Sequence[JobSource]:
|
|
"""List job-source records, optionally filtered by job."""
|
|
async with self._session_scope(session) as _session:
|
|
query = select(JobSource).options(
|
|
selectinload(JobSource.job), # pyright: ignore[reportArgumentType]
|
|
selectinload(JobSource.source), # pyright: ignore[reportArgumentType]
|
|
)
|
|
if job_id is not None:
|
|
query = query.where(JobSource.job_id == job_id)
|
|
result = await _session.exec(query)
|
|
return result.all()
|
|
|
|
async def update_job_source_transcription(
|
|
self,
|
|
*,
|
|
job_id: UUID,
|
|
source_id: UUID,
|
|
text: str | None,
|
|
error_detail: str | None = None,
|
|
ai_metadata: TranscriptionMetadata | dict[str, JsonValue] | None = None,
|
|
raw_api_response: dict[str, JsonValue] | None = None,
|
|
provider: str | None = None,
|
|
model: str | None = None,
|
|
request_manifest: RequestManifest | None = None,
|
|
transport_evidence: TransportEvidence | None = None,
|
|
failure_phase: str | None = None,
|
|
error_category: str | None = None,
|
|
started_at: datetime | None = None,
|
|
finished_at: datetime | None = None,
|
|
duration_ms: int | None = None,
|
|
session: AsyncSession | None = None,
|
|
) -> JobSource:
|
|
"""Persist transcription fields for one source within a specific job."""
|
|
async with self._session_scope(session) as _session:
|
|
job = await _session.get(Job, job_id)
|
|
if job is None:
|
|
raise TranscriptionNotFoundError(
|
|
f"Job with id {job_id} not found",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Verify the job id and retry.",
|
|
)
|
|
|
|
source = await _session.get(Source, source_id)
|
|
if source is None:
|
|
raise TranscriptionNotFoundError(
|
|
f"Source with id {source_id} not found",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Verify the source id and retry.",
|
|
)
|
|
|
|
if source.document_id != job.document_id:
|
|
raise TranscriptionError(
|
|
f"Source {source_id} does not belong to job {job_id}",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Link the source to the same document as the job and retry.",
|
|
)
|
|
|
|
job.provider = provider or job.provider or self.settings.provider.value
|
|
job.model = model or job.model or _resolve_transcript_model(provider=self.provider, settings=self.settings)
|
|
job.date_updated = datetime.now(UTC)
|
|
metadata_payload = _validate_transcription_metadata(ai_metadata)
|
|
raw_response_payload = _validate_json_object(raw_api_response, field_name="raw_api_response")
|
|
|
|
if text is not None:
|
|
source.raw_transcription = text
|
|
|
|
existing_job_source = await _session.exec(
|
|
select(JobSource).where(JobSource.job_id == job_id).where(JobSource.source_id == source_id)
|
|
)
|
|
job_source = existing_job_source.first()
|
|
if job_source is None:
|
|
job_source = JobSource(
|
|
job_id=job_id,
|
|
source_id=source_id,
|
|
status=JobSourceStatus.TRANSCRIBED if text is not None else JobSourceStatus.FAILED,
|
|
raw_transcription=text,
|
|
ai_metadata=metadata_payload,
|
|
raw_api_response=raw_response_payload,
|
|
error_detail=error_detail,
|
|
)
|
|
_session.add(job_source)
|
|
else:
|
|
job_source.raw_transcription = text
|
|
job_source.ai_metadata = metadata_payload
|
|
job_source.raw_api_response = raw_response_payload
|
|
job_source.error_detail = error_detail
|
|
job_source.status = JobSourceStatus.TRANSCRIBED if text is not None else JobSourceStatus.FAILED
|
|
job_source.executed_at = datetime.now(UTC)
|
|
|
|
finish_time = finished_at or datetime.now(UTC)
|
|
start_time = started_at or finish_time
|
|
attempt_number = (
|
|
await _session.exec(
|
|
select(func.max(ExecutionAttempt.attempt_number))
|
|
.where(ExecutionAttempt.job_id == job_id)
|
|
.where(ExecutionAttempt.source_id == source_id)
|
|
)
|
|
).one()
|
|
transport = transport_evidence or TransportEvidence(response_received=False)
|
|
manifest_payload = request_manifest.model_dump(mode="json") if request_manifest is not None else None
|
|
software_payload = (
|
|
request_manifest.software.model_dump(mode="json") if request_manifest is not None else None
|
|
)
|
|
attempt = ExecutionAttempt(
|
|
job_source_id=job_source.id,
|
|
job_id=job_id,
|
|
source_id=source_id,
|
|
attempt_number=(attempt_number or 0) + 1,
|
|
status=JobSourceStatus.TRANSCRIBED if text is not None else JobSourceStatus.FAILED,
|
|
provider=provider or job.provider or self.settings.provider.value,
|
|
model=model or job.model,
|
|
request_manifest=manifest_payload,
|
|
request_manifest_sha256=request_manifest.digest() if request_manifest is not None else None,
|
|
request_manifest_schema_version=(
|
|
request_manifest.schema_version if request_manifest is not None else None
|
|
),
|
|
response_received=transport.response_received,
|
|
transport_status_code=transport.status_code,
|
|
transport_body=transport.body,
|
|
transport_content_type=transport.content_type,
|
|
transport_content_encoding=transport.content_encoding,
|
|
transport_safe_headers=transport.safe_headers or None,
|
|
router_request_id=transport.request_id,
|
|
router_generation_id=transport.generation_id,
|
|
sdk_response_snapshot=raw_response_payload,
|
|
normalized_metadata=metadata_payload,
|
|
software_context=software_payload,
|
|
raw_transcription=text,
|
|
error_category=error_category,
|
|
error_detail=error_detail,
|
|
failure_phase=failure_phase,
|
|
started_at=start_time,
|
|
finished_at=finish_time,
|
|
duration_ms=duration_ms
|
|
if duration_ms is not None
|
|
else max(0, int((finish_time - start_time).total_seconds() * 1000)),
|
|
)
|
|
_session.add(attempt)
|
|
|
|
await self._finalize(session=_session, caller_session=session, refresh=(job, source, job_source, attempt))
|
|
return job_source
|
|
|
|
async def list_execution_attempts(
|
|
self,
|
|
*,
|
|
source_id: UUID | None = None,
|
|
job_id: UUID | None = None,
|
|
session: AsyncSession | None = None,
|
|
) -> Sequence[ExecutionAttempt]:
|
|
"""List immutable execution evidence in stable attempt order."""
|
|
async with self._session_scope(session) as _session:
|
|
query = select(ExecutionAttempt).options(selectinload(ExecutionAttempt.artifacts))
|
|
if source_id is not None:
|
|
query = query.where(ExecutionAttempt.source_id == source_id)
|
|
if job_id is not None:
|
|
query = query.where(ExecutionAttempt.job_id == job_id)
|
|
query = query.order_by(
|
|
ExecutionAttempt.job_id,
|
|
ExecutionAttempt.source_id,
|
|
ExecutionAttempt.attempt_number,
|
|
ExecutionAttempt.id,
|
|
)
|
|
return (await _session.exec(query)).all()
|
|
|
|
async def create_processing_artifact(
|
|
self,
|
|
artifact: ProcessingArtifact,
|
|
*,
|
|
session: AsyncSession | None = None,
|
|
) -> ProcessingArtifact:
|
|
"""Persist a validated generic processing artifact."""
|
|
if (artifact.inline_payload is None) == (artifact.external_reference is None):
|
|
raise TranscriptionError(
|
|
"Processing artifact requires exactly one content location",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Provide inline JSON or one stable external reference, but not both.",
|
|
)
|
|
self._verify_artifact_integrity(artifact)
|
|
async with self._session_scope(session) as _session:
|
|
_session.add(artifact)
|
|
await self._finalize(session=_session, caller_session=session)
|
|
return artifact
|
|
|
|
async def create_json_artifact(
|
|
self,
|
|
*,
|
|
source_id: UUID,
|
|
execution_attempt_id: UUID | None,
|
|
artifact_type: str,
|
|
schema_name: str,
|
|
schema_version: str,
|
|
producer: str,
|
|
producer_version: str,
|
|
payload: dict[str, JsonValue],
|
|
coordinate_metadata: dict[str, JsonValue] | None = None,
|
|
session: AsyncSession | None = None,
|
|
) -> ProcessingArtifact:
|
|
"""Store canonical JSON inline or atomically in the constrained artifact root."""
|
|
if coordinate_metadata is not None:
|
|
required = {"units", "origin", "width", "height", "transformations"}
|
|
missing = required.difference(coordinate_metadata)
|
|
if missing:
|
|
raise TranscriptionError(
|
|
f"Coordinate metadata is missing required fields: {', '.join(sorted(missing))}",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Declare units, origin, dimensions, and transformations.",
|
|
)
|
|
payload_bytes = canonical_json_bytes(payload)
|
|
artifact_id = uuid4()
|
|
inline_payload: dict[str, JsonValue] | None = payload
|
|
external_reference: str | None = None
|
|
external_path: Path | None = None
|
|
if len(payload_bytes) > self.settings.artifact_inline_threshold_bytes:
|
|
relative_path = Path(str(source_id)) / f"{artifact_id}.json"
|
|
external_path = self.settings.artifact_dir / relative_path
|
|
self._write_external_artifact(path=external_path, content=payload_bytes)
|
|
inline_payload = None
|
|
external_reference = relative_path.as_posix()
|
|
artifact = ProcessingArtifact(
|
|
id=artifact_id,
|
|
source_id=source_id,
|
|
execution_attempt_id=execution_attempt_id,
|
|
artifact_type=artifact_type,
|
|
media_type="application/json",
|
|
schema_name=schema_name,
|
|
schema_version=schema_version,
|
|
producer=producer,
|
|
producer_version=producer_version,
|
|
inline_payload=inline_payload,
|
|
external_reference=external_reference,
|
|
payload_sha256=hashlib.sha256(payload_bytes).hexdigest(),
|
|
byte_size=len(payload_bytes),
|
|
coordinate_metadata=coordinate_metadata,
|
|
)
|
|
try:
|
|
return await self.create_processing_artifact(artifact, session=session)
|
|
except Exception:
|
|
if external_path is not None:
|
|
external_path.unlink(missing_ok=True)
|
|
raise
|
|
|
|
def _write_external_artifact(self, *, path: Path, content: bytes) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary_path = path.with_suffix(f"{path.suffix}.tmp")
|
|
try:
|
|
with temporary_path.open("wb") as stream:
|
|
stream.write(content)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
temporary_path.replace(path)
|
|
except OSError as exc:
|
|
temporary_path.unlink(missing_ok=True)
|
|
raise TranscriptionError(
|
|
"Failed to persist external processing artifact",
|
|
category=ErrorCategory.INFRA_PERSISTENT,
|
|
suggestion="Verify artifact storage permissions and available disk space.",
|
|
) from exc
|
|
|
|
def _verify_external_artifact(self, artifact: ProcessingArtifact) -> None:
|
|
relative_path = Path(artifact.external_reference or "")
|
|
if relative_path.is_absolute() or ".." in relative_path.parts:
|
|
raise TranscriptionError(
|
|
"External artifact reference must stay inside the artifact root",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Use a relative application-managed artifact reference.",
|
|
)
|
|
artifact_root = self.settings.artifact_dir.resolve()
|
|
artifact_path = (artifact_root / relative_path).resolve()
|
|
if artifact_root not in artifact_path.parents or not artifact_path.is_file():
|
|
raise TranscriptionError(
|
|
"External processing artifact is unavailable",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Restore the artifact file or remove its pending database record.",
|
|
)
|
|
content = artifact_path.read_bytes()
|
|
if len(content) != artifact.byte_size or hashlib.sha256(content).hexdigest() != artifact.payload_sha256:
|
|
raise TranscriptionError(
|
|
"External processing artifact failed integrity verification",
|
|
category=ErrorCategory.CONFLICT,
|
|
suggestion="Restore the expected artifact bytes before retrying.",
|
|
)
|
|
|
|
def _verify_artifact_integrity(self, artifact: ProcessingArtifact) -> None:
|
|
if artifact.inline_payload is None:
|
|
self._verify_external_artifact(artifact)
|
|
return
|
|
content = canonical_json_bytes(artifact.inline_payload)
|
|
if len(content) != artifact.byte_size or hashlib.sha256(content).hexdigest() != artifact.payload_sha256:
|
|
raise TranscriptionError(
|
|
"Inline processing artifact failed integrity verification",
|
|
category=ErrorCategory.CONFLICT,
|
|
suggestion="Recreate the artifact with its canonical payload digest and byte size.",
|
|
)
|
|
|
|
async def list_processing_artifacts(
|
|
self,
|
|
*,
|
|
source_id: UUID,
|
|
session: AsyncSession | None = None,
|
|
) -> Sequence[ProcessingArtifact]:
|
|
"""List generic artifacts associated with a Source."""
|
|
async with self._session_scope(session) as _session:
|
|
query = (
|
|
select(ProcessingArtifact)
|
|
.where(ProcessingArtifact.source_id == source_id)
|
|
.order_by(ProcessingArtifact.created_at, ProcessingArtifact.id)
|
|
)
|
|
return (await _session.exec(query)).all()
|
|
|
|
async def list_processing_artifact_summaries(
|
|
self,
|
|
*,
|
|
source_id: UUID,
|
|
limit: int = 100,
|
|
session: AsyncSession | None = None,
|
|
) -> Sequence[ProcessingArtifact]:
|
|
"""List artifact metadata without loading potentially large inline payloads."""
|
|
async with self._session_scope(session) as _session:
|
|
query = (
|
|
select(ProcessingArtifact)
|
|
.options(defer(ProcessingArtifact.inline_payload)) # pyright: ignore[reportArgumentType]
|
|
.where(ProcessingArtifact.source_id == source_id)
|
|
.order_by(ProcessingArtifact.created_at, ProcessingArtifact.id)
|
|
.limit(limit)
|
|
)
|
|
return (await _session.exec(query)).all()
|
|
|
|
async def build_evidence_export(
|
|
self,
|
|
*,
|
|
source_id: UUID,
|
|
session: AsyncSession | None = None,
|
|
) -> dict[str, JsonValue]:
|
|
"""Build a versioned, source-reference-only evidence export."""
|
|
async with self._session_scope(session) as _session:
|
|
source = await _session.get(Source, source_id)
|
|
if source is None:
|
|
raise TranscriptionNotFoundError(
|
|
f"Source with id {source_id} not found",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Verify the source id and retry.",
|
|
)
|
|
attempts = list(await self.list_execution_attempts(source_id=source_id, session=_session))
|
|
artifacts = list(await self.list_processing_artifacts(source_id=source_id, session=_session))
|
|
|
|
for artifact in artifacts:
|
|
self._verify_artifact_integrity(artifact)
|
|
|
|
artifact_payloads = [
|
|
{
|
|
"id": str(artifact.id),
|
|
"source_id": str(artifact.source_id),
|
|
"execution_attempt_id": (str(artifact.execution_attempt_id) if artifact.execution_attempt_id else None),
|
|
"artifact_type": artifact.artifact_type,
|
|
"media_type": artifact.media_type,
|
|
"schema_name": artifact.schema_name,
|
|
"schema_version": artifact.schema_version,
|
|
"producer": artifact.producer,
|
|
"producer_version": artifact.producer_version,
|
|
"inline_payload": artifact.inline_payload,
|
|
"external_reference": artifact.external_reference,
|
|
"payload_sha256": artifact.payload_sha256,
|
|
"byte_size": artifact.byte_size,
|
|
"coordinate_metadata": artifact.coordinate_metadata,
|
|
"created_at": artifact.created_at.isoformat(),
|
|
}
|
|
for artifact in artifacts
|
|
]
|
|
attempt_payloads = [
|
|
{
|
|
"id": str(attempt.id),
|
|
"job_id": str(attempt.job_id),
|
|
"source_id": str(attempt.source_id),
|
|
"attempt_number": attempt.attempt_number,
|
|
"status": attempt.status.value,
|
|
"provider": attempt.provider,
|
|
"model": attempt.model,
|
|
"request_manifest": attempt.request_manifest,
|
|
"request_manifest_sha256": attempt.request_manifest_sha256,
|
|
"request_manifest_schema_version": attempt.request_manifest_schema_version,
|
|
"transport": {
|
|
"response_received": attempt.response_received,
|
|
"status_code": attempt.transport_status_code,
|
|
"body_base64": (
|
|
base64.b64encode(attempt.transport_body).decode("ascii")
|
|
if attempt.transport_body is not None
|
|
else None
|
|
),
|
|
"body_sha256": (
|
|
hashlib.sha256(attempt.transport_body).hexdigest()
|
|
if attempt.transport_body is not None
|
|
else None
|
|
),
|
|
"content_type": attempt.transport_content_type,
|
|
"content_encoding": attempt.transport_content_encoding,
|
|
"safe_headers": attempt.transport_safe_headers,
|
|
"request_id": attempt.router_request_id,
|
|
"generation_id": attempt.router_generation_id,
|
|
},
|
|
"sdk_response_snapshot": attempt.sdk_response_snapshot,
|
|
"normalized_metadata": attempt.normalized_metadata,
|
|
"software_context": attempt.software_context,
|
|
"raw_transcription": attempt.raw_transcription,
|
|
"error_category": attempt.error_category,
|
|
"error_detail": attempt.error_detail,
|
|
"failure_phase": attempt.failure_phase,
|
|
"started_at": attempt.started_at.isoformat(),
|
|
"finished_at": attempt.finished_at.isoformat(),
|
|
"duration_ms": attempt.duration_ms,
|
|
}
|
|
for attempt in attempts
|
|
]
|
|
return {
|
|
"schema_name": "transcription.evidence-export",
|
|
"schema_version": "1",
|
|
"source": {
|
|
"id": str(source.id),
|
|
"digest_sha256": source.file_hash,
|
|
"byte_size": source.file_size_bytes,
|
|
"page_number": source.page_number,
|
|
"upload_name": source.upload_name,
|
|
},
|
|
"attempts": attempt_payloads,
|
|
"artifacts": artifact_payloads,
|
|
}
|
|
|
|
async def upsert_revision_for_source(
|
|
self,
|
|
*,
|
|
source_id: UUID,
|
|
text: str,
|
|
session: AsyncSession | None = None,
|
|
) -> Source:
|
|
"""Persist a human revision on a source page."""
|
|
async with self._session_scope(session) as _session:
|
|
source = await _session.get(Source, source_id)
|
|
if source is None:
|
|
raise TranscriptionNotFoundError(
|
|
f"Source with id {source_id} not found",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Verify the source id and retry.",
|
|
)
|
|
|
|
source.revised_text = text
|
|
source.date_revised = datetime.now(UTC)
|
|
await self._finalize(session=_session, caller_session=session, refresh=(source,))
|
|
return source
|
|
|
|
async def read_revision_by_source(
|
|
self,
|
|
source_id: UUID,
|
|
*,
|
|
session: AsyncSession | None = None,
|
|
) -> Source | None:
|
|
"""Read the source record for a given page, including any revision text."""
|
|
async with self._session_scope(session) as _session:
|
|
return await _session.get(Source, source_id)
|
|
|
|
async def list_revisions_by_job(
|
|
self,
|
|
job_id: UUID,
|
|
*,
|
|
session: AsyncSession | None = None,
|
|
) -> Sequence[Source]:
|
|
"""List source pages for a job that carry revision text."""
|
|
async with self._session_scope(session) as _session:
|
|
query = (
|
|
select(Source)
|
|
.join(JobSource, JobSource.source_id == Source.id)
|
|
.where(JobSource.job_id == job_id)
|
|
.where(Source.revised_text.is_not(None))
|
|
.order_by(Source.date_revised) # pyright: ignore[reportArgumentType]
|
|
)
|
|
result = await _session.exec(query)
|
|
return result.all()
|
|
|
|
|
|
def _resolve_transcript_model(*, provider: TranscriptionProvider, settings: Settings) -> str:
|
|
provider_model = getattr(provider, "model", None)
|
|
if isinstance(provider_model, str) and provider_model.strip():
|
|
return provider_model
|
|
|
|
if settings.provider_model and settings.provider_model.strip():
|
|
return settings.provider_model
|
|
|
|
return "unknown"
|
|
|
|
|
|
def _validate_transcription_metadata(
|
|
metadata: TranscriptionMetadata | dict[str, JsonValue] | None,
|
|
) -> dict[str, JsonValue] | None:
|
|
if metadata is None:
|
|
return None
|
|
try:
|
|
validated = (
|
|
metadata if isinstance(metadata, TranscriptionMetadata) else TranscriptionMetadata.model_validate(metadata)
|
|
)
|
|
except ValidationError as exc:
|
|
raise TranscriptionError(
|
|
"Transcription metadata failed validation",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Persist only normalized provider execution metadata.",
|
|
) from exc
|
|
return validated.as_json_object()
|
|
|
|
|
|
def _validate_json_object(
|
|
payload: dict[str, JsonValue] | None,
|
|
*,
|
|
field_name: str,
|
|
) -> dict[str, JsonValue] | None:
|
|
if payload is None:
|
|
return None
|
|
try:
|
|
return JSON_OBJECT_ADAPTER.validate_python(payload)
|
|
except ValidationError as exc:
|
|
raise TranscriptionError(
|
|
f"{field_name} must be a JSON-compatible object",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Remove non-JSON values before persisting provider diagnostics.",
|
|
) from exc
|
|
|
|
|
|
def hash_prompt_text(prompt_text: str) -> str:
|
|
"""Return the canonical SHA-256 provenance hash for prompt text."""
|
|
return hashlib.sha256(prompt_text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
async def transcribe_document_image(
|
|
image_path: str | Path,
|
|
*,
|
|
prompt_name: str | None = None,
|
|
prompt_text: str | None = None,
|
|
temperature: float | None = None,
|
|
top_p: float | None = None,
|
|
settings: Settings | None = None,
|
|
provider: TranscriptionProvider | None = None,
|
|
source_reference: SourceEvidenceReference | None = None,
|
|
) -> TranscriptionResult:
|
|
"""Transcribe a local image using the configured prompt and provider."""
|
|
runtime_settings = settings or get_settings()
|
|
if prompt_text is None:
|
|
prompt_execution = build_prompt_execution(prompt_name=prompt_name, settings=runtime_settings)
|
|
else:
|
|
effective_prompt_name = (prompt_name or runtime_settings.default_prompt_name or DEFAULT_PROMPT_FILE).strip()
|
|
prompt_execution = PromptExecution(
|
|
prompt_name=effective_prompt_name,
|
|
prompt_hash=hash_prompt_text(prompt_text),
|
|
system_prompt=None,
|
|
user_prompt=prompt_text,
|
|
temperature=temperature if temperature is not None else runtime_settings.transcription_temperature,
|
|
top_p=top_p if top_p is not None else runtime_settings.transcription_top_p,
|
|
)
|
|
image_bytes, mime_type = load_source_payload(image_path)
|
|
|
|
owns_adapter = provider is None
|
|
adapter = provider or get_transcription_provider(settings=runtime_settings)
|
|
logger.info("Starting transcription for image=%s mime_type=%s", image_path, mime_type)
|
|
|
|
try:
|
|
with handle_transcription_errors():
|
|
result = await adapter.transcribe(
|
|
prompt_text=prompt_execution.user_prompt,
|
|
image_bytes=image_bytes,
|
|
mime_type=mime_type,
|
|
temperature=prompt_execution.temperature,
|
|
top_p=prompt_execution.top_p,
|
|
source_reference=source_reference,
|
|
)
|
|
finally:
|
|
if owns_adapter:
|
|
close = getattr(adapter, "aclose", None)
|
|
if close is not None:
|
|
await close()
|
|
logger.info("Transcription completed for image=%s provider=%s", image_path, result.provider)
|
|
return TranscriptionResult(
|
|
text=result.text,
|
|
provider=result.provider,
|
|
prompt_name=prompt_execution.prompt_name,
|
|
prompt_hash=prompt_execution.prompt_hash,
|
|
system_prompt=prompt_execution.system_prompt,
|
|
user_prompt=result.user_prompt or prompt_execution.user_prompt,
|
|
temperature=result.temperature if result.temperature is not None else prompt_execution.temperature,
|
|
top_p=result.top_p if result.top_p is not None else prompt_execution.top_p,
|
|
model=result.model,
|
|
metadata=result.metadata,
|
|
raw_api_response=result.raw_api_response,
|
|
request_manifest=result.request_manifest,
|
|
transport_evidence=result.transport_evidence,
|
|
)
|
|
|
|
|
|
def build_prompt_execution(*, prompt_name: str | None = None, settings: Settings | None = None) -> PromptExecution:
|
|
"""Resolve the exact prompt payload and provenance for one execution."""
|
|
runtime_settings = settings or get_settings()
|
|
effective_prompt_name = (prompt_name or runtime_settings.default_prompt_name or DEFAULT_PROMPT_FILE).strip()
|
|
user_prompt = load_prompt_text(prompt_name=effective_prompt_name, settings=runtime_settings)
|
|
return PromptExecution(
|
|
prompt_name=effective_prompt_name,
|
|
prompt_hash=hash_prompt_text(user_prompt),
|
|
system_prompt=None,
|
|
user_prompt=user_prompt,
|
|
temperature=runtime_settings.transcription_temperature,
|
|
top_p=runtime_settings.transcription_top_p,
|
|
)
|
|
|
|
|
|
def load_prompt_text(*, prompt_name: str = DEFAULT_PROMPT_FILE, settings: Settings | None = None) -> str:
|
|
"""Load and validate prompt text from PROMPT_DIR."""
|
|
runtime_settings = settings or get_settings()
|
|
prompt_root = runtime_settings.prompt_dir.resolve()
|
|
prompt_path = (prompt_root / prompt_name).resolve()
|
|
if prompt_path.parent != prompt_root:
|
|
raise PromptLoadError(
|
|
f"Prompt file must be directly inside PROMPT_DIR: {prompt_name}",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Configure a prompt filename without directory components.",
|
|
)
|
|
|
|
if not prompt_path.exists() or not prompt_path.is_file():
|
|
raise PromptLoadError(
|
|
f"Prompt file not found: {prompt_path}",
|
|
category=ErrorCategory.INFRA_PERSISTENT,
|
|
suggestion="Verify PROMPT_DIR and prompt file configuration, then retry.",
|
|
)
|
|
|
|
prompt_text = prompt_path.read_text(encoding="utf-8").strip()
|
|
if not prompt_text:
|
|
raise PromptLoadError(
|
|
f"Prompt file is empty: {prompt_path}",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Populate the prompt file with valid instructions and retry.",
|
|
)
|
|
|
|
logger.info("Loaded prompt artifact: %s", prompt_path)
|
|
return prompt_text
|
|
|
|
|
|
def source_mime_type(filename: str | Path) -> str:
|
|
"""Return the canonical MIME type for a supported Source filename."""
|
|
path = Path(filename)
|
|
suffix = path.suffix.lower()
|
|
mime_type = SOURCE_MIME_TYPES.get(suffix)
|
|
if mime_type is None:
|
|
supported = ", ".join(sorted(SOURCE_EXTENSIONS))
|
|
raise TranscriptionError(
|
|
f"Unsupported Source format: {suffix or '<none>'}",
|
|
category=ErrorCategory.USER_INPUT,
|
|
suggestion=f"Use one of the supported Source formats: {supported}.",
|
|
)
|
|
return mime_type
|
|
|
|
|
|
def validate_source_content(*, filename: str | Path, content: bytes) -> str:
|
|
"""Validate Source content and return its canonical MIME type."""
|
|
if not content:
|
|
raise TranscriptionError(
|
|
"Source content is empty",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Select a non-empty Source file and try again.",
|
|
)
|
|
|
|
safe_name = Path(filename).name
|
|
if not safe_name:
|
|
raise TranscriptionError(
|
|
"Source filename is required",
|
|
category=ErrorCategory.VALIDATION,
|
|
suggestion="Choose a Source file with a valid filename and retry.",
|
|
)
|
|
return source_mime_type(safe_name)
|
|
|
|
|
|
def load_source_payload(source_path: str | Path) -> tuple[bytes, str]:
|
|
"""Read Source bytes and resolve MIME type from the canonical format policy."""
|
|
path = Path(source_path)
|
|
|
|
if not path.exists() or not path.is_file():
|
|
raise TranscriptionError(
|
|
f"Source file not found: {path}",
|
|
category=ErrorCategory.NOT_FOUND,
|
|
suggestion="Verify the Source file exists and retry from the jobs page.",
|
|
)
|
|
|
|
content = path.read_bytes()
|
|
return content, validate_source_content(filename=path.name, content=content)
|
|
|
|
|
|
@contextmanager
|
|
def handle_transcription_errors():
|
|
"""Context manager to handle transcription errors."""
|
|
try:
|
|
yield
|
|
except ProviderAuthError as exc:
|
|
raise TranscriptionError(
|
|
"Provider authentication failed",
|
|
category=ErrorCategory.INFRA_PERSISTENT,
|
|
suggestion="Verify provider API credentials and retry.",
|
|
) from exc
|
|
except ProviderResponseError as exc:
|
|
raise TranscriptionError(
|
|
"Provider returned an invalid response",
|
|
category=ErrorCategory.EXTERNAL_PROVIDER,
|
|
suggestion="Retry once. If it persists, try a different provider model or inspect provider status.",
|
|
retriable=True,
|
|
) from exc
|
|
except ProviderError as exc:
|
|
raise TranscriptionError(
|
|
f"Provider transcription failed: {exc}",
|
|
category=ErrorCategory.EXTERNAL_PROVIDER,
|
|
suggestion="Retry the transcription from jobs. If repeated, check provider availability.",
|
|
retriable=True,
|
|
) from exc
|