V3 step 1 update models.py and step 2 implement service/worker, and raw API response persistence

This commit is contained in:
Jim Lancaster
2026-08-08 18:08:04 -05:00
parent 5a741de0a9
commit 4dac9349c1
22 changed files with 373 additions and 51 deletions
+3
View File
@@ -69,6 +69,9 @@ class Settings(BaseSettings):
provider_model: str | None = None provider_model: str | None = None
openrouter_http_referer: str | None = None openrouter_http_referer: str | None = None
openrouter_app_title: str | None = None openrouter_app_title: str | None = None
default_prompt_name: str = "transcribe_document.md"
transcription_temperature: float | None = None
transcription_top_p: float | None = None
# --- runtime environment --- # --- runtime environment ---
environment: Literal["development", "test", "production"] = "development" environment: Literal["development", "test", "production"] = "development"
+10 -2
View File
@@ -1,4 +1,4 @@
"""SQLModel domain models for the V2 transcription system.""" """SQLModel domain models for the V3 transcription system."""
from datetime import UTC from datetime import UTC
from datetime import date from datetime import date
@@ -10,6 +10,7 @@ from uuid import UUID
from uuid import uuid4 from uuid import uuid4
from sqlalchemy import Column from sqlalchemy import Column
from sqlalchemy import BigInteger
from sqlalchemy import JSON from sqlalchemy import JSON
from sqlalchemy import UniqueConstraint from sqlalchemy import UniqueConstraint
from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.dialects.postgresql import JSONB
@@ -125,7 +126,6 @@ class Job(SQLModel, table=True):
date_updated: datetime = Field(default_factory=lambda: datetime.now(UTC)) date_updated: datetime = Field(default_factory=lambda: datetime.now(UTC))
provider: str | None = None provider: str | None = None
model: str | None = None model: str | None = None
prompt_name: str | None = None
document: Optional["Document"] = Relationship(back_populates="jobs", sa_relationship_kwargs={"lazy": "selectin"}) document: Optional["Document"] = Relationship(back_populates="jobs", sa_relationship_kwargs={"lazy": "selectin"})
job_sources: list["JobSource"] = Relationship(back_populates="job", sa_relationship_kwargs={"lazy": "selectin"}) job_sources: list["JobSource"] = Relationship(back_populates="job", sa_relationship_kwargs={"lazy": "selectin"})
@@ -173,6 +173,8 @@ class Source(SQLModel, table=True):
upload_name: str upload_name: str
filename: str filename: str
file_path: str file_path: str
file_hash: str
file_size_bytes: int = Field(sa_column=Column(BigInteger(), nullable=False))
raw_transcription: str | None = None raw_transcription: str | None = None
revised_text: str | None = None revised_text: str | None = None
date_uploaded: datetime = Field(default_factory=lambda: datetime.now(UTC)) date_uploaded: datetime = Field(default_factory=lambda: datetime.now(UTC))
@@ -221,6 +223,12 @@ class JobSource(SQLModel, table=True):
source_id: UUID = Field(foreign_key="source.id") source_id: UUID = Field(foreign_key="source.id")
status: JobSourceStatus = Field(default=JobSourceStatus.PENDING) status: JobSourceStatus = Field(default=JobSourceStatus.PENDING)
raw_transcription: str | None = None raw_transcription: str | None = None
prompt_name: str | None = None
prompt_hash: str | None = None
system_prompt: str | None = None
user_prompt: str | None = None
temperature: float | None = None
top_p: float | None = None
ai_metadata: dict[str, Any] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True)) ai_metadata: dict[str, Any] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True))
raw_api_response: dict[str, Any] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True)) raw_api_response: dict[str, Any] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True))
error_detail: str | None = None error_detail: str | None = None
+14 -1
View File
@@ -25,6 +25,11 @@ class TranscriptionResult:
provider: str provider: str
prompt_name: str prompt_name: str
model: str model: str
prompt_hash: str | None = None
system_prompt: str | None = None
user_prompt: str | None = None
temperature: float | None = None
top_p: float | None = None
finish_reason: str | None = None finish_reason: str | None = None
usage_input_tokens: int | None = None usage_input_tokens: int | None = None
usage_output_tokens: int | None = None usage_output_tokens: int | None = None
@@ -36,6 +41,14 @@ class TranscriptionResult:
class TranscriptionProvider(Protocol): class TranscriptionProvider(Protocol):
"""Contract every transcription provider adapter must satisfy.""" """Contract every transcription provider adapter must satisfy."""
async def transcribe(self, *, prompt_text: str, image_bytes: bytes, mime_type: str) -> TranscriptionResult: async def transcribe(
self,
*,
prompt_text: str,
image_bytes: bytes,
mime_type: str,
temperature: float | None = None,
top_p: float | None = None,
) -> TranscriptionResult:
"""Transcribe the provided image according to the prompt text.""" """Transcribe the provided image according to the prompt text."""
... ...
+36 -3
View File
@@ -31,6 +31,8 @@ class OpenRouterRequest:
messages: list[dict[str, Any]] messages: list[dict[str, Any]]
http_referer: str | None http_referer: str | None
x_open_router_title: str | None x_open_router_title: str | None
temperature: float | None
top_p: float | None
class OpenRouterTranscriptionProvider: class OpenRouterTranscriptionProvider:
@@ -46,15 +48,31 @@ class OpenRouterTranscriptionProvider:
"""Return the resolved OpenRouter model slug.""" """Return the resolved OpenRouter model slug."""
return self._model return self._model
async def transcribe(self, *, prompt_text: str, image_bytes: bytes, mime_type: str) -> TranscriptionResult: async def transcribe(
self,
*,
prompt_text: str,
image_bytes: bytes,
mime_type: str,
temperature: float | None = None,
top_p: float | None = None,
) -> TranscriptionResult:
"""Send prompt + image to OpenRouter and return normalized text output.""" """Send prompt + image to OpenRouter and return normalized text output."""
request = self._build_request(prompt_text=prompt_text, image_bytes=image_bytes, mime_type=mime_type) request = self._build_request(
prompt_text=prompt_text,
image_bytes=image_bytes,
mime_type=mime_type,
temperature=temperature,
top_p=top_p,
)
try: try:
response = await self._client.chat.send_async( response = await self._client.chat.send_async(
messages=cast(list[ChatMessagesTypedDict], request.messages), messages=cast(list[ChatMessagesTypedDict], request.messages),
model=request.model, model=request.model,
http_referer=request.http_referer, http_referer=request.http_referer,
x_open_router_title=request.x_open_router_title, x_open_router_title=request.x_open_router_title,
temperature=request.temperature,
top_p=request.top_p,
) )
except Exception as exc: except Exception as exc:
message = str(exc).lower() message = str(exc).lower()
@@ -78,6 +96,11 @@ class OpenRouterTranscriptionProvider:
text=text, text=text,
provider="openrouter", provider="openrouter",
prompt_name="", prompt_name="",
prompt_hash=None,
system_prompt=None,
user_prompt=prompt_text,
temperature=temperature,
top_p=top_p,
model=model, model=model,
finish_reason=finish_reason, finish_reason=finish_reason,
usage_input_tokens=usage_input_tokens, usage_input_tokens=usage_input_tokens,
@@ -148,7 +171,15 @@ class OpenRouterTranscriptionProvider:
return repr(value) return repr(value)
def _build_request(self, *, prompt_text: str, image_bytes: bytes, mime_type: str) -> OpenRouterRequest: def _build_request(
self,
*,
prompt_text: str,
image_bytes: bytes,
mime_type: str,
temperature: float | None,
top_p: float | None,
) -> OpenRouterRequest:
image_b64 = base64.b64encode(image_bytes).decode("ascii") image_b64 = base64.b64encode(image_bytes).decode("ascii")
data_url = f"data:{mime_type};base64,{image_b64}" data_url = f"data:{mime_type};base64,{image_b64}"
@@ -167,6 +198,8 @@ class OpenRouterTranscriptionProvider:
messages=messages, messages=messages,
http_referer=self._settings.openrouter_http_referer, http_referer=self._settings.openrouter_http_referer,
x_open_router_title=self._settings.openrouter_app_title, x_open_router_title=self._settings.openrouter_app_title,
temperature=temperature,
top_p=top_p,
) )
def _extract_text(self, response: Any) -> str: def _extract_text(self, response: Any) -> str:
+22 -4
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
from collections.abc import Sequence from collections.abc import Sequence
from dataclasses import dataclass from dataclasses import dataclass
import hashlib
import logging import logging
from pathlib import Path from pathlib import Path
from uuid import UUID from uuid import UUID
@@ -48,6 +49,8 @@ class PendingStoredUpload:
source_id: UUID source_id: UUID
original_filename: str original_filename: str
stored_path: Path stored_path: Path
file_hash: str
file_size_bytes: int
async def create_upload_job( async def create_upload_job(
@@ -68,6 +71,7 @@ async def create_upload_job(
relative_directory=Path("documents") / str(document_id), relative_directory=Path("documents") / str(document_id),
filename_stem=str(source_id), filename_stem=str(source_id),
) )
file_hash, file_size_bytes = _compute_file_metadata(file_bytes)
try: try:
document, job = await _create_upload_records( document, job = await _create_upload_records(
session=session, session=session,
@@ -75,6 +79,8 @@ async def create_upload_job(
source_id=source_id, source_id=source_id,
original_filename=filename, original_filename=filename,
stored_path=stored_path, stored_path=stored_path,
file_hash=file_hash,
file_size_bytes=file_size_bytes,
) )
except Exception as exc: except Exception as exc:
_best_effort_delete(stored_path) _best_effort_delete(stored_path)
@@ -101,7 +107,6 @@ async def create_job_for_document(
session: AsyncSession, session: AsyncSession,
provider: str | None = None, provider: str | None = None,
model: str | None = None, model: str | None = None,
prompt_name: str | None = None,
settings: Settings | None = None, settings: Settings | None = None,
) -> JobCreateResult: ) -> JobCreateResult:
"""Create a queued job for an existing document with one or more uploaded sources.""" """Create a queued job for an existing document with one or more uploaded sources."""
@@ -128,6 +133,8 @@ async def create_job_for_document(
relative_directory=Path("documents") / str(document_id), relative_directory=Path("documents") / str(document_id),
filename_stem=str(source_id), filename_stem=str(source_id),
), ),
file_hash=_compute_file_hash(file_bytes),
file_size_bytes=len(file_bytes),
) )
) )
@@ -138,7 +145,6 @@ async def create_job_for_document(
stored_uploads=stored_uploads, stored_uploads=stored_uploads,
provider=provider, provider=provider,
model=model, model=model,
prompt_name=prompt_name,
) )
except Exception as exc: except Exception as exc:
for upload in stored_uploads: for upload in stored_uploads:
@@ -165,6 +171,8 @@ async def _create_upload_records(
source_id: UUID, source_id: UUID,
original_filename: str, original_filename: str,
stored_path: Path, stored_path: Path,
file_hash: str,
file_size_bytes: int,
) -> tuple[Document, Job]: ) -> tuple[Document, Job]:
document = Document( document = Document(
id=document_id, id=document_id,
@@ -184,6 +192,8 @@ async def _create_upload_records(
upload_name=Path(original_filename).name, upload_name=Path(original_filename).name,
filename=stored_path.name, filename=stored_path.name,
file_path=str(stored_path), file_path=str(stored_path),
file_hash=file_hash,
file_size_bytes=file_size_bytes,
) )
session.add(source) session.add(source)
await session.flush() await session.flush()
@@ -209,7 +219,6 @@ async def _create_job_for_document_records(
stored_uploads: Sequence[PendingStoredUpload], stored_uploads: Sequence[PendingStoredUpload],
provider: str | None, provider: str | None,
model: str | None, model: str | None,
prompt_name: str | None,
) -> tuple[Job, list[UUID]]: ) -> tuple[Job, list[UUID]]:
document = await session.get(Document, document_id) document = await session.get(Document, document_id)
if document is None: if document is None:
@@ -228,7 +237,6 @@ async def _create_job_for_document_records(
document_id=document_id, document_id=document_id,
provider=(provider or None), provider=(provider or None),
model=(model or None), model=(model or None),
prompt_name=(prompt_name or None),
) )
session.add(job) session.add(job)
await session.flush() await session.flush()
@@ -242,6 +250,8 @@ async def _create_job_for_document_records(
upload_name=Path(upload.original_filename).name, upload_name=Path(upload.original_filename).name,
filename=upload.stored_path.name, filename=upload.stored_path.name,
file_path=str(upload.stored_path), file_path=str(upload.stored_path),
file_hash=upload.file_hash,
file_size_bytes=upload.file_size_bytes,
) )
session.add(source) session.add(source)
await session.flush() await session.flush()
@@ -268,6 +278,14 @@ def _best_effort_delete(path: Path) -> None:
logger.warning("Failed to clean up upload file after DB error: %s", path) logger.warning("Failed to clean up upload file after DB error: %s", path)
def _compute_file_hash(file_bytes: bytes) -> str:
return hashlib.sha256(file_bytes).hexdigest()
def _compute_file_metadata(file_bytes: bytes) -> tuple[str, int]:
return _compute_file_hash(file_bytes), len(file_bytes)
def store_file( def store_file(
*, *,
filename: str, filename: str,
+84 -6
View File
@@ -2,6 +2,8 @@
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass
import hashlib
import logging import logging
import mimetypes import mimetypes
from collections.abc import Sequence from collections.abc import Sequence
@@ -39,6 +41,18 @@ DEFAULT_PROMPT_FILE = "transcribe_document.md"
SUPPORTED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".tif", ".tiff", ".pdf"} SUPPORTED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".tif", ".tiff", ".pdf"}
@dataclass(frozen=True)
class PromptExecution:
"""Resolved prompt inputs captured for one page execution."""
prompt_name: str
prompt_hash: str
system_prompt: str | None
user_prompt: str
temperature: float | None
top_p: float | None
class PromptLoadError(AppError): class PromptLoadError(AppError):
"""Raised when prompt artifacts cannot be loaded safely.""" """Raised when prompt artifacts cannot be loaded safely."""
@@ -354,6 +368,11 @@ class TranscriptionService(ServiceBase):
provider=result.provider, provider=result.provider,
model=result.model, model=result.model,
prompt_name=result.prompt_name, prompt_name=result.prompt_name,
prompt_hash=result.prompt_hash,
system_prompt=result.system_prompt,
user_prompt=result.user_prompt,
temperature=result.temperature,
top_p=result.top_p,
session=session, session=session,
) )
@@ -366,6 +385,11 @@ class TranscriptionService(ServiceBase):
provider: str | None = None, provider: str | None = None,
model: str | None = None, model: str | None = None,
prompt_name: str = DEFAULT_PROMPT_FILE, prompt_name: str = DEFAULT_PROMPT_FILE,
prompt_hash: str | None = None,
system_prompt: str | None = None,
user_prompt: str | None = None,
temperature: float | None = None,
top_p: float | None = None,
session: AsyncSession | None = None, session: AsyncSession | None = None,
) -> Job: ) -> Job:
"""Persist transcription output for the first ordered source in a job's document. """Persist transcription output for the first ordered source in a job's document.
@@ -384,7 +408,6 @@ class TranscriptionService(ServiceBase):
job.provider = provider or job.provider or self.settings.provider.value job.provider = provider or job.provider or self.settings.provider.value
job.model = model or job.model or _resolve_transcript_model(provider=self.provider, settings=self.settings) job.model = model or job.model or _resolve_transcript_model(provider=self.provider, settings=self.settings)
job.prompt_name = prompt_name or job.prompt_name or DEFAULT_PROMPT_FILE
job.date_updated = datetime.now(UTC) job.date_updated = datetime.now(UTC)
source = await _session.exec( source = await _session.exec(
@@ -402,6 +425,11 @@ class TranscriptionService(ServiceBase):
provider=provider, provider=provider,
model=model, model=model,
prompt_name=prompt_name, prompt_name=prompt_name,
prompt_hash=prompt_hash,
system_prompt=system_prompt,
user_prompt=user_prompt,
temperature=temperature,
top_p=top_p,
session=_session, session=_session,
) )
@@ -420,6 +448,11 @@ class TranscriptionService(ServiceBase):
provider: str | None = None, provider: str | None = None,
model: str | None = None, model: str | None = None,
prompt_name: str = DEFAULT_PROMPT_FILE, prompt_name: str = DEFAULT_PROMPT_FILE,
prompt_hash: str | None = None,
system_prompt: str | None = None,
user_prompt: str | None = None,
temperature: float | None = None,
top_p: float | None = None,
session: AsyncSession | None = None, session: AsyncSession | None = None,
) -> JobSource: ) -> JobSource:
"""Persist transcription fields for one source within a specific job.""" """Persist transcription fields for one source within a specific job."""
@@ -449,9 +482,9 @@ class TranscriptionService(ServiceBase):
job.provider = provider or job.provider or self.settings.provider.value job.provider = provider or job.provider or self.settings.provider.value
job.model = model or job.model or _resolve_transcript_model(provider=self.provider, settings=self.settings) job.model = model or job.model or _resolve_transcript_model(provider=self.provider, settings=self.settings)
job.prompt_name = prompt_name or job.prompt_name or DEFAULT_PROMPT_FILE
job.date_updated = datetime.now(UTC) job.date_updated = datetime.now(UTC)
if text is not None:
source.raw_transcription = text source.raw_transcription = text
existing_job_source = await _session.exec( existing_job_source = await _session.exec(
@@ -464,6 +497,12 @@ class TranscriptionService(ServiceBase):
source_id=source_id, source_id=source_id,
status=JobSourceStatus.TRANSCRIBED if text is not None else JobSourceStatus.FAILED, status=JobSourceStatus.TRANSCRIBED if text is not None else JobSourceStatus.FAILED,
raw_transcription=text, raw_transcription=text,
prompt_name=prompt_name,
prompt_hash=prompt_hash,
system_prompt=system_prompt,
user_prompt=user_prompt,
temperature=temperature,
top_p=top_p,
ai_metadata=ai_metadata, ai_metadata=ai_metadata,
raw_api_response=raw_api_response, raw_api_response=raw_api_response,
error_detail=error_detail, error_detail=error_detail,
@@ -471,6 +510,12 @@ class TranscriptionService(ServiceBase):
_session.add(job_source) _session.add(job_source)
else: else:
job_source.raw_transcription = text job_source.raw_transcription = text
job_source.prompt_name = prompt_name
job_source.prompt_hash = prompt_hash
job_source.system_prompt = system_prompt
job_source.user_prompt = user_prompt
job_source.temperature = temperature
job_source.top_p = top_p
job_source.ai_metadata = ai_metadata job_source.ai_metadata = ai_metadata
job_source.raw_api_response = raw_api_response job_source.raw_api_response = raw_api_response
job_source.error_detail = error_detail job_source.error_detail = error_detail
@@ -545,13 +590,13 @@ def _resolve_transcript_model(*, provider: TranscriptionProvider, settings: Sett
async def transcribe_document_image( async def transcribe_document_image(
image_path: str | Path, image_path: str | Path,
*, *,
prompt_name: str = DEFAULT_PROMPT_FILE, prompt_name: str | None = None,
settings: Settings | None = None, settings: Settings | None = None,
provider: TranscriptionProvider | None = None, provider: TranscriptionProvider | None = None,
) -> TranscriptionResult: ) -> TranscriptionResult:
"""Transcribe a local image using the configured prompt and provider.""" """Transcribe a local image using the configured prompt and provider."""
runtime_settings = settings or get_settings() runtime_settings = settings or get_settings()
prompt_text = load_prompt_text(prompt_name=prompt_name, settings=runtime_settings) prompt_execution = build_prompt_execution(prompt_name=prompt_name, settings=runtime_settings)
image_bytes, mime_type = load_image_payload(image_path) image_bytes, mime_type = load_image_payload(image_path)
adapter = provider or get_transcription_provider(settings=runtime_settings) adapter = provider or get_transcription_provider(settings=runtime_settings)
@@ -559,12 +604,45 @@ async def transcribe_document_image(
with handle_transcription_errors(): with handle_transcription_errors():
result = await adapter.transcribe( result = await adapter.transcribe(
prompt_text=prompt_text, prompt_text=prompt_execution.user_prompt,
image_bytes=image_bytes, image_bytes=image_bytes,
mime_type=mime_type, mime_type=mime_type,
temperature=prompt_execution.temperature,
top_p=prompt_execution.top_p,
) )
logger.info("Transcription completed for image=%s provider=%s", image_path, result.provider) logger.info("Transcription completed for image=%s provider=%s", image_path, result.provider)
return result return TranscriptionResult(
text=result.text,
provider=result.provider,
prompt_name=prompt_execution.prompt_name,
prompt_hash=prompt_execution.prompt_hash,
system_prompt=prompt_execution.system_prompt,
user_prompt=result.user_prompt or prompt_execution.user_prompt,
temperature=result.temperature if result.temperature is not None else prompt_execution.temperature,
top_p=result.top_p if result.top_p is not None else prompt_execution.top_p,
model=result.model,
finish_reason=result.finish_reason,
usage_input_tokens=result.usage_input_tokens,
usage_output_tokens=result.usage_output_tokens,
usage_total_tokens=result.usage_total_tokens,
ai_metadata=result.ai_metadata,
raw_api_response=result.raw_api_response,
)
def build_prompt_execution(*, prompt_name: str | None = None, settings: Settings | None = None) -> PromptExecution:
"""Resolve the exact prompt payload and provenance for one execution."""
runtime_settings = settings or get_settings()
effective_prompt_name = (prompt_name or runtime_settings.default_prompt_name or DEFAULT_PROMPT_FILE).strip()
user_prompt = load_prompt_text(prompt_name=effective_prompt_name, settings=runtime_settings)
return PromptExecution(
prompt_name=effective_prompt_name,
prompt_hash=hashlib.sha256(user_prompt.encode("utf-8")).hexdigest(),
system_prompt=None,
user_prompt=user_prompt,
temperature=runtime_settings.transcription_temperature,
top_p=runtime_settings.transcription_top_p,
)
def load_prompt_text(*, prompt_name: str = DEFAULT_PROMPT_FILE, settings: Settings | None = None) -> str: def load_prompt_text(*, prompt_name: str = DEFAULT_PROMPT_FILE, settings: Settings | None = None) -> str:
+73 -14
View File
@@ -16,6 +16,7 @@ from ..errors import format_error_detail
from ..providers import TranscriptionResult from ..providers import TranscriptionResult
from . import ServiceBundle from . import ServiceBundle
from .transcription import DEFAULT_PROMPT_FILE from .transcription import DEFAULT_PROMPT_FILE
from .transcription import build_prompt_execution
from .transcription import transcribe_document_image from .transcription import transcribe_document_image
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -84,11 +85,13 @@ async def process_queued_job(
if not sources: if not sources:
return await services.jobs.mark_job_status(job.id, JobStatus.TRANSCRIBED, session=session) return await services.jobs.mark_job_status(job.id, JobStatus.TRANSCRIBED, session=session)
successful_pages: list[tuple[Source, TranscriptionResult]] = [] successful_pages: list[tuple[Source, TranscriptionResult, str, str | None, str, float | None, float | None]] = []
failed_pages: list[tuple[Source, AppError]] = [] failed_pages: list[tuple[Source, AppError, str, str, str | None, str, float | None, float | None]] = []
externally_stopped = False externally_stopped = False
for source in sources: for source in sources:
prompt_execution = build_prompt_execution(settings=runtime_settings)
if await _job_no_longer_processing(job_id=job.id, services=services, session=session): if await _job_no_longer_processing(job_id=job.id, services=services, session=session):
externally_stopped = True externally_stopped = True
break break
@@ -96,7 +99,11 @@ async def process_queued_job(
started_at = asyncio.get_running_loop().time() started_at = asyncio.get_running_loop().time()
try: try:
result = await asyncio.wait_for( result = await asyncio.wait_for(
transcribe_document_image(source.file_path), transcribe_document_image(
source.file_path,
settings=runtime_settings,
provider=services.transcriptions.provider,
),
timeout=runtime_settings.worker_provider_timeout_seconds, timeout=runtime_settings.worker_provider_timeout_seconds,
) )
elapsed_seconds = asyncio.get_running_loop().time() - started_at elapsed_seconds = asyncio.get_running_loop().time() - started_at
@@ -120,7 +127,17 @@ async def process_queued_job(
) )
_validate_transcription_quality(result=result, settings=runtime_settings) _validate_transcription_quality(result=result, settings=runtime_settings)
successful_pages.append((source, result)) successful_pages.append(
(
source,
result,
prompt_execution.prompt_hash,
prompt_execution.system_prompt,
prompt_execution.user_prompt,
prompt_execution.temperature,
prompt_execution.top_p,
)
)
except TimeoutError: except TimeoutError:
error = AppError( error = AppError(
f"Provider call timed out after {runtime_settings.worker_provider_timeout_seconds:.1f}s", f"Provider call timed out after {runtime_settings.worker_provider_timeout_seconds:.1f}s",
@@ -128,7 +145,18 @@ async def process_queued_job(
suggestion="Retry the job. If this repeats, verify provider latency and request payload size.", suggestion="Retry the job. If this repeats, verify provider latency and request payload size.",
retriable=True, retriable=True,
) )
failed_pages.append((source, error)) failed_pages.append(
(
source,
error,
prompt_execution.prompt_name,
prompt_execution.prompt_hash,
prompt_execution.system_prompt,
prompt_execution.user_prompt,
prompt_execution.temperature,
prompt_execution.top_p,
)
)
logger.error( logger.error(
"Source failed operation=worker.process_job job_id=%s document_id=%s source_id=%s error_id=%s category=%s", "Source failed operation=worker.process_job job_id=%s document_id=%s source_id=%s error_id=%s category=%s",
job.id, job.id,
@@ -144,7 +172,18 @@ async def process_queued_job(
case _: case _:
error = classify_unexpected_error(exc, operation="worker.process_job") error = classify_unexpected_error(exc, operation="worker.process_job")
failed_pages.append((source, error)) failed_pages.append(
(
source,
error,
prompt_execution.prompt_name,
prompt_execution.prompt_hash,
prompt_execution.system_prompt,
prompt_execution.user_prompt,
prompt_execution.temperature,
prompt_execution.top_p,
)
)
logger.error( logger.error(
"Source failed operation=worker.process_job job_id=%s document_id=%s source_id=%s error_id=%s category=%s", "Source failed operation=worker.process_job job_id=%s document_id=%s source_id=%s error_id=%s category=%s",
job.id, job.id,
@@ -368,15 +407,15 @@ async def _finalize_batch_outcome(
*, *,
job: Job, job: Job,
services: ServiceBundle, services: ServiceBundle,
successful_pages: list[tuple[Source, TranscriptionResult]], successful_pages: list[tuple[Source, TranscriptionResult, str, str | None, str, float | None, float | None]],
failed_pages: list[tuple[Source, AppError]], failed_pages: list[tuple[Source, AppError, str, str, str | None, str, float | None, float | None]],
status: JobStatus, status: JobStatus,
session: AsyncSession | None = None, session: AsyncSession | None = None,
) -> Job: ) -> Job:
"""Transaction B: write per-source outcomes and terminal job status atomically.""" """Transaction B: write per-source outcomes and terminal job status atomically."""
if session is None: if session is None:
async with services.jobs._session_scope() as local_session: async with services.jobs._session_scope() as local_session:
for source, result in successful_pages: for source, result, prompt_hash, system_prompt, user_prompt, temperature, top_p in successful_pages:
await services.transcriptions.update_job_source_transcription( await services.transcriptions.update_job_source_transcription(
job_id=job.id, job_id=job.id,
source_id=source.id, source_id=source.id,
@@ -387,16 +426,26 @@ async def _finalize_batch_outcome(
provider=result.provider, provider=result.provider,
model=result.model, model=result.model,
prompt_name=result.prompt_name, prompt_name=result.prompt_name,
prompt_hash=result.prompt_hash or prompt_hash,
system_prompt=result.system_prompt if result.system_prompt is not None else system_prompt,
user_prompt=result.user_prompt if result.user_prompt is not None else user_prompt,
temperature=result.temperature if result.temperature is not None else temperature,
top_p=result.top_p if result.top_p is not None else top_p,
session=local_session, session=local_session,
) )
for source, error in failed_pages: for source, error, prompt_name, prompt_hash, system_prompt, user_prompt, temperature, top_p in failed_pages:
await services.transcriptions.update_job_source_transcription( await services.transcriptions.update_job_source_transcription(
job_id=job.id, job_id=job.id,
source_id=source.id, source_id=source.id,
text=None, text=None,
error_detail=format_error_detail(error), error_detail=format_error_detail(error),
prompt_name=DEFAULT_PROMPT_FILE, prompt_name=prompt_name,
prompt_hash=prompt_hash,
system_prompt=system_prompt,
user_prompt=user_prompt,
temperature=temperature,
top_p=top_p,
session=local_session, session=local_session,
) )
@@ -404,7 +453,7 @@ async def _finalize_batch_outcome(
await local_session.commit() await local_session.commit()
return updated_job return updated_job
for source, result in successful_pages: for source, result, prompt_hash, system_prompt, user_prompt, temperature, top_p in successful_pages:
await services.transcriptions.update_job_source_transcription( await services.transcriptions.update_job_source_transcription(
job_id=job.id, job_id=job.id,
source_id=source.id, source_id=source.id,
@@ -415,16 +464,26 @@ async def _finalize_batch_outcome(
provider=result.provider, provider=result.provider,
model=result.model, model=result.model,
prompt_name=result.prompt_name, prompt_name=result.prompt_name,
prompt_hash=result.prompt_hash or prompt_hash,
system_prompt=result.system_prompt if result.system_prompt is not None else system_prompt,
user_prompt=result.user_prompt if result.user_prompt is not None else user_prompt,
temperature=result.temperature if result.temperature is not None else temperature,
top_p=result.top_p if result.top_p is not None else top_p,
session=session, session=session,
) )
for source, error in failed_pages: for source, error, prompt_name, prompt_hash, system_prompt, user_prompt, temperature, top_p in failed_pages:
await services.transcriptions.update_job_source_transcription( await services.transcriptions.update_job_source_transcription(
job_id=job.id, job_id=job.id,
source_id=source.id, source_id=source.id,
text=None, text=None,
error_detail=format_error_detail(error), error_detail=format_error_detail(error),
prompt_name=DEFAULT_PROMPT_FILE, prompt_name=prompt_name,
prompt_hash=prompt_hash,
system_prompt=system_prompt,
user_prompt=user_prompt,
temperature=temperature,
top_p=top_p,
session=session, session=session,
) )
@@ -29,7 +29,7 @@ def render_original_transcription_card(*, job: Job, classes: str = "w-full") ->
with card, ui.column().classes("w-full q-gutter-y-sm"): with card, ui.column().classes("w-full q-gutter-y-sm"):
ui.label(header).classes("text-subtitle1 text-weight-medium") ui.label(header).classes("text-subtitle1 text-weight-medium")
ui.label(caption).classes("text-caption vibe-text-muted") ui.label(caption).classes("text-caption vibe-text-muted")
_metadata_row(label="Prompt", value=job.prompt_name or "unknown") _metadata_row(label="Prompt", value=_latest_job_prompt(job) or "unknown")
_metadata_row(label="Updated", value=_format_created_at(job.date_updated)) _metadata_row(label="Updated", value=_format_created_at(job.date_updated))
latest_transcription = _latest_job_transcription(job) latest_transcription = _latest_job_transcription(job)
@@ -114,6 +114,13 @@ def _latest_job_error_detail(job: Job) -> str | None:
return None return None
def _latest_job_prompt(job: Job) -> str | None:
for job_source in sorted(job.job_sources, key=lambda item: item.executed_at, reverse=True):
if job_source.prompt_name:
return job_source.prompt_name
return None
def _format_created_at(value: datetime) -> str: def _format_created_at(value: datetime) -> str:
"""Return a compact UTC-like timestamp for row captions.""" """Return a compact UTC-like timestamp for row captions."""
return value.strftime("%Y-%m-%d %H:%M:%S %Z") return value.strftime("%Y-%m-%d %H:%M:%S %Z")
+10 -4
View File
@@ -91,10 +91,9 @@ def register_page() -> None: # noqa: PLR0915
if requested_document_id in document_options: if requested_document_id in document_options:
document_select.value = requested_document_id document_select.value = requested_document_id
with ui.row().classes("w-full gap-3 grid grid-cols-1 md:grid-cols-3"): with ui.row().classes("w-full gap-3 grid grid-cols-1 md:grid-cols-2"):
provider_input = ui.input(label="Provider").props("outlined").classes("ui-form-surface") provider_input = ui.input(label="Provider").props("outlined").classes("ui-form-surface")
model_input = ui.input(label="Model").props("outlined").classes("ui-form-surface") model_input = ui.input(label="Model").props("outlined").classes("ui-form-surface")
prompt_input = ui.input(label="Prompt").props("outlined").classes("ui-form-surface")
_render_upload_section(uploaded_files) _render_upload_section(uploaded_files)
@@ -119,7 +118,6 @@ def register_page() -> None: # noqa: PLR0915
uploads=uploaded_files, uploads=uploaded_files,
provider=(provider_input.value or None), provider=(provider_input.value or None),
model=(model_input.value or None), model=(model_input.value or None),
prompt_name=(prompt_input.value or None),
session=session, session=session,
) )
except Exception as exc: # noqa: BLE001 except Exception as exc: # noqa: BLE001
@@ -421,7 +419,7 @@ def _render_job_logistics(job: Job) -> None:
with archival_card(title="Execution Logistics"): with archival_card(title="Execution Logistics"):
metadata_row("Provider:", job.provider or "pending") metadata_row("Provider:", job.provider or "pending")
metadata_row("Model:", job.model or "pending") metadata_row("Model:", job.model or "pending")
metadata_row("Prompt:", job.prompt_name or "pending") metadata_row("Prompt:", _latest_prompt_name(job) or "pending")
metadata_row("Retry Count:", str(job.retry_count)) metadata_row("Retry Count:", str(job.retry_count))
metadata_row("Last Updated:", job.date_updated.isoformat()) metadata_row("Last Updated:", job.date_updated.isoformat())
@@ -449,3 +447,11 @@ def _parse_uuid(value: str | None) -> UUID | None:
return UUID(value) return UUID(value)
except ValueError: except ValueError:
return None return None
def _latest_prompt_name(job: Job) -> str | None:
ordered = sorted(job.job_sources, key=lambda item: item.executed_at, reverse=True)
for job_source in ordered:
if job_source.prompt_name:
return job_source.prompt_name
return None
+1 -1
View File
@@ -281,7 +281,7 @@ def _render_source_job_metadata_zone(latest_job_source: JobSource | None) -> Non
) )
metadata_row( metadata_row(
"Prompt:", "Prompt:",
latest_job_source.job.prompt_name if latest_job_source.job and latest_job_source.job.prompt_name else "unknown", latest_job_source.prompt_name or "unknown",
) )
if latest_job_source.error_detail: if latest_job_source.error_detail:
+16 -6
View File
@@ -46,7 +46,13 @@ class TestPipelineSuccessFlow:
self, async_session, default_session_factory, tmp_path: Path, monkeypatch self, async_session, default_session_factory, tmp_path: Path, monkeypatch
): ):
"""Upload followed by worker processing persists job transcription and transcribed status.""" """Upload followed by worker processing persists job transcription and transcribed status."""
settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path) settings = Settings(
openrouter_api_key="test-key",
upload_dir=tmp_path,
default_prompt_name="transcribe_document.md",
transcription_temperature=0.2,
transcription_top_p=0.85,
)
upload_result = await create_upload_job( upload_result = await create_upload_job(
filename="pipeline.jpg", filename="pipeline.jpg",
file_bytes=b"pipeline-bytes", file_bytes=b"pipeline-bytes",
@@ -89,13 +95,17 @@ class TestPipelineSuccessFlow:
queued_job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session) queued_job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
processed = queued_job is not None processed = queued_job is not None
if queued_job is not None: if queued_job is not None:
await advance_job(job=queued_job, services=services, session=async_session) await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session) job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
assert processed is True assert processed is True
assert job is not None assert job is not None
assert job.status == JobStatus.TRANSCRIBED assert job.status == JobStatus.TRANSCRIBED
assert any(job_source.raw_transcription == "Pipeline transcript" for job_source in job.job_sources) assert any(job_source.raw_transcription == "Pipeline transcript" for job_source in job.job_sources)
assert any(job_source.prompt_name == "transcribe_document.md" for job_source in job.job_sources)
assert any(job_source.user_prompt is not None for job_source in job.job_sources)
assert any(job_source.temperature == 0.2 for job_source in job.job_sources)
assert any(job_source.top_p == 0.85 for job_source in job.job_sources)
assert any(job_source.ai_metadata == {"finish_reason": "stop", "usage": {"total_tokens": 42}} for job_source in job.job_sources) assert any(job_source.ai_metadata == {"finish_reason": "stop", "usage": {"total_tokens": 42}} for job_source in job.job_sources)
assert any( assert any(
job_source.raw_api_response == {"id": "resp_123", "choices": [{"message": {"content": "Pipeline transcript"}}]} job_source.raw_api_response == {"id": "resp_123", "choices": [{"message": {"content": "Pipeline transcript"}}]}
@@ -153,7 +163,7 @@ class TestPipelineSuccessFlow:
queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session) queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
assert queued_job is not None assert queued_job is not None
await advance_job(job=queued_job, services=services, session=async_session) await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session) job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
assert job.status == JobStatus.TRANSCRIBED assert job.status == JobStatus.TRANSCRIBED
@@ -216,7 +226,7 @@ class TestPipelineSuccessFlow:
queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session) queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
assert queued_job is not None assert queued_job is not None
await advance_job(job=queued_job, services=services, session=async_session) await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session) job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
assert job.status == JobStatus.PARTIAL_SUCCESS assert job.status == JobStatus.PARTIAL_SUCCESS
@@ -289,7 +299,7 @@ class TestPipelineSuccessFlow:
queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session) queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
assert queued_job is not None assert queued_job is not None
await advance_job(job=queued_job, services=services, session=async_session) await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
refreshed = await services.jobs.read_job(job_id=create_result.job_id, session=async_session) refreshed = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
assert call_count == 1 assert call_count == 1
@@ -337,7 +347,7 @@ class TestPipelineFailureFlow:
queued_job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session) queued_job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
processed = queued_job is not None processed = queued_job is not None
if queued_job is not None: if queued_job is not None:
await advance_job(job=queued_job, services=services, session=async_session) await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session) job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
assert processed is True assert processed is True
+21
View File
@@ -73,6 +73,27 @@ class TestOpenRouterProviderTranscribe:
assert send_call["x_open_router_title"] == "Transcription App" assert send_call["x_open_router_title"] == "Transcription App"
assert result.text == "Transcript text" assert result.text == "Transcript text"
@pytest.mark.asyncio
async def test_includes_temperature_and_top_p_when_provided(self):
"""Transcribe passes configured sampling parameters through to OpenRouter."""
response = {"model": "vendor/model-a", "choices": [{"message": {"content": "Transcript text"}}]}
client = _FakeClient(response=response)
provider = OpenRouterTranscriptionProvider(settings=Settings(openrouter_api_key="test-key"), client=client)
result = await provider.transcribe(
prompt_text="Prompt body",
image_bytes=b"img-bytes",
mime_type="image/png",
temperature=0.2,
top_p=0.85,
)
send_call = client.chat.calls[0]
assert send_call["temperature"] == 0.2
assert send_call["top_p"] == 0.85
assert result.temperature == 0.2
assert result.top_p == 0.85
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_parses_successful_response_text(self): async def test_parses_successful_response_text(self):
"""Transcribe returns normalized text from a valid response payload.""" """Transcribe returns normalized text from a valid response payload."""
+2
View File
@@ -78,6 +78,8 @@ async def test_delete_document_blocks_when_dependencies_exist(default_session_fa
upload_name="001_page.png", upload_name="001_page.png",
filename="001_page.png", filename="001_page.png",
file_path="uploads/001_page.png", file_path="uploads/001_page.png",
file_hash="a" * 64,
file_size_bytes=1,
) )
) )
session.add(Job(document_id=document.id)) session.add(Job(document_id=document.id))
+15 -3
View File
@@ -80,6 +80,8 @@ class TestJobService:
upload_name="letter.jpg", upload_name="letter.jpg",
filename="stored-letter.jpg", filename="stored-letter.jpg",
file_path="/uploads/stored-letter.jpg", file_path="/uploads/stored-letter.jpg",
file_hash="a" * 64,
file_size_bytes=1,
) )
session.add(source) session.add(source)
await session.flush() await session.flush()
@@ -120,7 +122,7 @@ class TestJobService:
assert next_job.id == first.id assert next_job.id == first.id
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_create_job_persists_provider_model_prompt( async def test_create_job_persists_provider_and_model(
self, self,
job_service: JobService, job_service: JobService,
document_service: DocumentService, document_service: DocumentService,
@@ -132,14 +134,12 @@ class TestJobService:
document_id=document.id, document_id=document.id,
provider="openrouter", provider="openrouter",
model="google/gemini-2.5-flash", model="google/gemini-2.5-flash",
prompt_name="transcribe_document.md",
) )
await job_service.create_job(job=job) await job_service.create_job(job=job)
fetched = await job_service.read_job(job_id=job.id) fetched = await job_service.read_job(job_id=job.id)
assert fetched.provider == "openrouter" assert fetched.provider == "openrouter"
assert fetched.model == "google/gemini-2.5-flash" assert fetched.model == "google/gemini-2.5-flash"
assert fetched.prompt_name == "transcribe_document.md"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_read_job_resolves_filename_from_linked_source( async def test_read_job_resolves_filename_from_linked_source(
@@ -160,6 +160,8 @@ class TestJobService:
upload_name="page_001.png", upload_name="page_001.png",
filename="stored_page_001.png", filename="stored_page_001.png",
file_path="/uploads/stored_page_001.png", file_path="/uploads/stored_page_001.png",
file_hash="b" * 64,
file_size_bytes=1,
) )
session.add(source) session.add(source)
await session.flush() await session.flush()
@@ -210,6 +212,8 @@ class TestJobService:
upload_name="delete-job-source.jpg", upload_name="delete-job-source.jpg",
filename="stored-delete-job-source.jpg", filename="stored-delete-job-source.jpg",
file_path="/uploads/stored-delete-job-source.jpg", file_path="/uploads/stored-delete-job-source.jpg",
file_hash="c" * 64,
file_size_bytes=1,
) )
session.add(source) session.add(source)
await session.flush() await session.flush()
@@ -246,6 +250,8 @@ class TestJobService:
upload_name="cancel-1.jpg", upload_name="cancel-1.jpg",
filename="stored-cancel-1.jpg", filename="stored-cancel-1.jpg",
file_path="/uploads/stored-cancel-1.jpg", file_path="/uploads/stored-cancel-1.jpg",
file_hash="d" * 64,
file_size_bytes=1,
) )
source_two = Source( source_two = Source(
document_id=document.id, document_id=document.id,
@@ -253,6 +259,8 @@ class TestJobService:
upload_name="cancel-2.jpg", upload_name="cancel-2.jpg",
filename="stored-cancel-2.jpg", filename="stored-cancel-2.jpg",
file_path="/uploads/stored-cancel-2.jpg", file_path="/uploads/stored-cancel-2.jpg",
file_hash="e" * 64,
file_size_bytes=1,
) )
session.add(source_one) session.add(source_one)
session.add(source_two) session.add(source_two)
@@ -304,6 +312,8 @@ class TestJobService:
upload_name="resubmit-1.jpg", upload_name="resubmit-1.jpg",
filename="stored-resubmit-1.jpg", filename="stored-resubmit-1.jpg",
file_path="/uploads/stored-resubmit-1.jpg", file_path="/uploads/stored-resubmit-1.jpg",
file_hash="f" * 64,
file_size_bytes=1,
raw_transcription="existing text", raw_transcription="existing text",
) )
source_two = Source( source_two = Source(
@@ -312,6 +322,8 @@ class TestJobService:
upload_name="resubmit-2.jpg", upload_name="resubmit-2.jpg",
filename="stored-resubmit-2.jpg", filename="stored-resubmit-2.jpg",
file_path="/uploads/stored-resubmit-2.jpg", file_path="/uploads/stored-resubmit-2.jpg",
file_hash="0" * 64,
file_size_bytes=1,
raw_transcription="done text", raw_transcription="done text",
) )
session.add(source_one) session.add(source_one)
+8 -2
View File
@@ -48,7 +48,6 @@ async def test_create_job_for_document_sorts_uploads_and_creates_links(async_ses
], ],
provider="openrouter", provider="openrouter",
model="test-model", model="test-model",
prompt_name="transcribe_document.md",
session=async_session, session=async_session,
settings=settings, settings=settings,
) )
@@ -57,7 +56,6 @@ async def test_create_job_for_document_sorts_uploads_and_creates_links(async_ses
assert created_job is not None assert created_job is not None
assert created_job.provider == "openrouter" assert created_job.provider == "openrouter"
assert created_job.model == "test-model" assert created_job.model == "test-model"
assert created_job.prompt_name == "transcribe_document.md"
sources = ( sources = (
await async_session.exec( await async_session.exec(
@@ -71,10 +69,16 @@ async def test_create_job_for_document_sorts_uploads_and_creates_links(async_ses
assert all("A_page" not in source.filename and "b_page" not in source.filename for source in sources) assert all("A_page" not in source.filename and "b_page" not in source.filename for source in sources)
assert all(Path(source.filename).stem == str(source.id) for source in sources) assert all(Path(source.filename).stem == str(source.id) for source in sources)
assert all(Path(source.file_path).parent == (tmp_path / "documents" / str(document.id)) for source in sources) assert all(Path(source.file_path).parent == (tmp_path / "documents" / str(document.id)) for source in sources)
assert [source.file_hash for source in sources] == [
"ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb",
"3e23e8160039594a33894f6564e1b1348bbd7a0088d42c4acb73eeaed59c009d",
]
assert [source.file_size_bytes for source in sources] == [1, 1]
job_sources = (await async_session.exec(select(JobSource).where(JobSource.job_id == result.job_id))).all() job_sources = (await async_session.exec(select(JobSource).where(JobSource.job_id == result.job_id))).all()
assert len(job_sources) == 2 assert len(job_sources) == 2
assert set(result.source_ids) == {job_source.source_id for job_source in job_sources} assert set(result.source_ids) == {job_source.source_id for job_source in job_sources}
assert {job_source.prompt_name for job_source in job_sources} == {None}
@pytest.mark.asyncio @pytest.mark.asyncio
@@ -103,6 +107,8 @@ async def test_create_upload_job_stores_source_under_document_id_directory(async
assert Path(source.filename).stem == str(source.id) assert Path(source.filename).stem == str(source.id)
assert result.stored_path.name == source.filename assert result.stored_path.name == source.filename
assert Path(source.file_path).parent == expected_parent assert Path(source.file_path).parent == expected_parent
assert source.file_hash == "2c8648d103e3dd7ad87660da0f126a1443b6d21ac1bd3ec000c5e24e2373a90c"
assert source.file_size_bytes == len(b"image-bytes")
def test_store_person_portrait_stores_file_under_person_id_directory(tmp_path): def test_store_person_portrait_stores_file_under_person_id_directory(tmp_path):
@@ -39,6 +39,8 @@ class TestTranscriptionServiceRevisionUpsert:
upload_name="source.jpg", upload_name="source.jpg",
filename="source.jpg", filename="source.jpg",
file_path="uploads/source.jpg", file_path="uploads/source.jpg",
file_hash="1" * 64,
file_size_bytes=1,
) )
async with transcriptions._session_scope() as session: async with transcriptions._session_scope() as session:
session.add(source) session.add(source)
@@ -74,6 +76,8 @@ class TestTranscriptionServiceRevisionUpsert:
upload_name="source.jpg", upload_name="source.jpg",
filename="source.jpg", filename="source.jpg",
file_path="uploads/source.jpg", file_path="uploads/source.jpg",
file_hash="2" * 64,
file_size_bytes=1,
) )
async with transcriptions._session_scope() as session: async with transcriptions._session_scope() as session:
session.add(source) session.add(source)
@@ -115,6 +119,8 @@ class TestTranscriptionServiceRevisionUpsert:
upload_name="delete.jpg", upload_name="delete.jpg",
filename="delete.jpg", filename="delete.jpg",
file_path=str(stored_path), file_path=str(stored_path),
file_hash="3" * 64,
file_size_bytes=4,
) )
async with transcriptions._session_scope() as session: async with transcriptions._session_scope() as session:
session.add(source) session.add(source)
@@ -149,6 +155,8 @@ class TestTranscriptionServiceRevisionUpsert:
upload_name="shared.jpg", upload_name="shared.jpg",
filename="shared.jpg", filename="shared.jpg",
file_path="uploads/shared.jpg", file_path="uploads/shared.jpg",
file_hash="4" * 64,
file_size_bytes=1,
) )
async with transcriptions._session_scope() as session: async with transcriptions._session_scope() as session:
session.add(source) session.add(source)
@@ -180,6 +188,8 @@ class TestTranscriptionServiceRevisionUpsert:
upload_name="orphan.jpg", upload_name="orphan.jpg",
filename="orphan.jpg", filename="orphan.jpg",
file_path=str(stored_path), file_path=str(stored_path),
file_hash="5" * 64,
file_size_bytes=4,
) )
await transcriptions.create_source(source=source) await transcriptions.create_source(source=source)
@@ -207,6 +217,8 @@ class TestTranscriptionServiceRevisionUpsert:
upload_name="linked.jpg", upload_name="linked.jpg",
filename="linked.jpg", filename="linked.jpg",
file_path="uploads/linked.jpg", file_path="uploads/linked.jpg",
file_hash="6" * 64,
file_size_bytes=1,
) )
async with transcriptions._session_scope() as session: async with transcriptions._session_scope() as session:
session.add(source) session.add(source)
+11
View File
@@ -60,6 +60,8 @@ async def test_transcription_service_manages_source_crud(default_session_factory
upload_name="page-1.jpg", upload_name="page-1.jpg",
filename="page-1.jpg", filename="page-1.jpg",
file_path="uploads/page-1.jpg", file_path="uploads/page-1.jpg",
file_hash="7" * 64,
file_size_bytes=1,
) )
) )
@@ -101,6 +103,8 @@ async def test_transcription_service_job_source_crud_uses_caller_session(default
upload_name="job-source.jpg", upload_name="job-source.jpg",
filename="job-source.jpg", filename="job-source.jpg",
file_path="uploads/job-source.jpg", file_path="uploads/job-source.jpg",
file_hash="8" * 64,
file_size_bytes=1,
) )
session.add(source) session.add(source)
await session.flush() await session.flush()
@@ -165,6 +169,8 @@ async def test_document_delete_is_blocked_with_source_and_job_dependencies(defau
upload_name="blocked.jpg", upload_name="blocked.jpg",
filename="blocked.jpg", filename="blocked.jpg",
file_path="uploads/blocked.jpg", file_path="uploads/blocked.jpg",
file_hash="9" * 64,
file_size_bytes=1,
) )
) )
await transcriptions.create_job_source( await transcriptions.create_job_source(
@@ -200,6 +206,8 @@ async def test_source_delete_blocks_when_linked_to_multiple_jobs(default_session
upload_name="shared-page.jpg", upload_name="shared-page.jpg",
filename="shared-page.jpg", filename="shared-page.jpg",
file_path="uploads/shared-page.jpg", file_path="uploads/shared-page.jpg",
file_hash="a" * 64,
file_size_bytes=1,
) )
) )
await transcriptions.create_job_source( await transcriptions.create_job_source(
@@ -228,6 +236,8 @@ async def test_update_job_source_transcription_persists_provider_json_payloads(d
upload_name="provider.jpg", upload_name="provider.jpg",
filename="provider.jpg", filename="provider.jpg",
file_path="uploads/provider.jpg", file_path="uploads/provider.jpg",
file_hash="b" * 64,
file_size_bytes=1,
) )
) )
await transcriptions.create_job_source( await transcriptions.create_job_source(
@@ -251,5 +261,6 @@ async def test_update_job_source_transcription_persists_provider_json_payloads(d
stored_rows = await transcriptions.list_job_sources(job_id=job.id) stored_rows = await transcriptions.list_job_sources(job_id=job.id)
assert len(stored_rows) == 1 assert len(stored_rows) == 1
assert stored_rows[0].raw_transcription == "provider transcript" assert stored_rows[0].raw_transcription == "provider transcript"
assert stored_rows[0].prompt_name == "transcribe_document.md"
assert stored_rows[0].ai_metadata == metadata assert stored_rows[0].ai_metadata == metadata
assert stored_rows[0].raw_api_response == raw_payload assert stored_rows[0].raw_api_response == raw_payload
+7 -1
View File
@@ -8,6 +8,8 @@ import pytest
from transcription.config import Settings from transcription.config import Settings
from transcription.db.models import Document from transcription.db.models import Document
from transcription.db.models import Job from transcription.db.models import Job
from transcription.db.models import JobSource
from transcription.db.models import JobSourceStatus
from transcription.db.models import JobStatus from transcription.db.models import JobStatus
from transcription.db.models import Source from transcription.db.models import Source
from transcription.services import ServiceBundle from transcription.services import ServiceBundle
@@ -49,12 +51,16 @@ class TestWorkflowReliability:
source = Source( source = Source(
document_id=document.id, document_id=document.id,
job_id=job.id, page_number=1,
upload_name="timeout.jpg", upload_name="timeout.jpg",
filename="timeout.jpg", filename="timeout.jpg",
file_path=str(Path("tests/fixtures/images/real/Book Two - page 02.jpg")), file_path=str(Path("tests/fixtures/images/real/Book Two - page 02.jpg")),
file_hash="c" * 64,
file_size_bytes=1,
) )
session.add(source) session.add(source)
await session.flush()
session.add(JobSource(job_id=job.id, source_id=source.id, status=JobSourceStatus.PENDING))
await session.commit() await session.commit()
loaded = await services.jobs.read_job(job_id=job.id, session=session) loaded = await services.jobs.read_job(job_id=job.id, session=session)
+2
View File
@@ -64,6 +64,8 @@ def _persist_source(session, document: Document, *, page_number: int = 1, **over
"raw_transcription": "Original machine text", "raw_transcription": "Original machine text",
} }
defaults.update(overrides) defaults.update(overrides)
defaults["file_hash"] = "a" * 64
defaults["file_size_bytes"] = 123
source = Source(**defaults) source = Source(**defaults)
session.add(source) session.add(source)
session.commit() session.commit()
+4 -1
View File
@@ -95,16 +95,18 @@ async def seed_job(app_client: tuple[FastAPI, TestClient]) -> Callable[..., Awai
retry_count=0, retry_count=0,
provider="openrouter", provider="openrouter",
model="google/gemini-2.5-flash", model="google/gemini-2.5-flash",
prompt_name="transcribe_document.md",
) )
session.add(job) session.add(job)
await session.flush() await session.flush()
source = Source( source = Source(
document_id=document.id, document_id=document.id,
page_number=1,
upload_name=filename, upload_name=filename,
filename=filename, filename=filename,
file_path=str(stored_path), file_path=str(stored_path),
file_hash="b" * 64,
file_size_bytes=len(stored_path.read_bytes()),
) )
session.add(source) session.add(source)
await session.flush() await session.flush()
@@ -119,6 +121,7 @@ async def seed_job(app_client: tuple[FastAPI, TestClient]) -> Callable[..., Awai
if transcription_text is not None if transcription_text is not None
else JobSourceStatus.FAILED else JobSourceStatus.FAILED
), ),
prompt_name="transcribe_document.md",
raw_transcription=transcription_text, raw_transcription=transcription_text,
error_detail=error_detail, error_detail=error_detail,
) )
+2
View File
@@ -146,6 +146,8 @@ class TestDocumentsPageRendering:
upload_name="page_1.png", upload_name="page_1.png",
filename="page_1.png", filename="page_1.png",
file_path="/tmp/page_1.png", file_path="/tmp/page_1.png",
file_hash="0" * 64,
file_size_bytes=1,
) )
session.add(source) session.add(source)
await session.commit() await session.commit()
+10
View File
@@ -23,6 +23,8 @@ class TestSourceModelProperties:
upload_name="page_one.png", upload_name="page_one.png",
filename="stored_page_one.png", filename="stored_page_one.png",
file_path="/tmp/stored_page_one.png", file_path="/tmp/stored_page_one.png",
file_hash="b" * 64,
file_size_bytes=1,
) )
assert source.latest_job_source is None assert source.latest_job_source is None
@@ -86,6 +88,8 @@ class TestSourcesPageRendering:
upload_name="page_one.png", upload_name="page_one.png",
filename="stored_page_one.png", filename="stored_page_one.png",
file_path="/tmp/stored_page_one.png", file_path="/tmp/stored_page_one.png",
file_hash="c" * 64,
file_size_bytes=1,
) )
) )
await session.commit() await session.commit()
@@ -114,6 +118,8 @@ class TestSourcesPageRendering:
upload_name="target_page.png", upload_name="target_page.png",
filename="target_stored.png", filename="target_stored.png",
file_path="/tmp/target_stored.png", file_path="/tmp/target_stored.png",
file_hash="d" * 64,
file_size_bytes=1,
), ),
Source( Source(
document_id=other.id, document_id=other.id,
@@ -121,6 +127,8 @@ class TestSourcesPageRendering:
upload_name="other_page.png", upload_name="other_page.png",
filename="other_stored.png", filename="other_stored.png",
file_path="/tmp/other_stored.png", file_path="/tmp/other_stored.png",
file_hash="e" * 64,
file_size_bytes=1,
), ),
] ]
) )
@@ -240,6 +248,8 @@ class TestSourcesPageRendering:
upload_name="orphan-source.png", upload_name="orphan-source.png",
filename="orphan-source.png", filename="orphan-source.png",
file_path="/tmp/orphan-source.png", file_path="/tmp/orphan-source.png",
file_hash="f" * 64,
file_size_bytes=1,
) )
session.add(source) session.add(source)
await session.commit() await session.commit()