V3 step 1 update models.py and step 2 implement service/worker, and raw API response persistence

This commit is contained in:
Jim Lancaster
2026-08-08 18:08:04 -05:00
parent 5a741de0a9
commit 4dac9349c1
22 changed files with 373 additions and 51 deletions
+16 -6
View File
@@ -46,7 +46,13 @@ class TestPipelineSuccessFlow:
self, async_session, default_session_factory, tmp_path: Path, monkeypatch
):
"""Upload followed by worker processing persists job transcription and transcribed status."""
settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
settings = Settings(
openrouter_api_key="test-key",
upload_dir=tmp_path,
default_prompt_name="transcribe_document.md",
transcription_temperature=0.2,
transcription_top_p=0.85,
)
upload_result = await create_upload_job(
filename="pipeline.jpg",
file_bytes=b"pipeline-bytes",
@@ -89,13 +95,17 @@ class TestPipelineSuccessFlow:
queued_job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
processed = queued_job is not None
if queued_job is not None:
await advance_job(job=queued_job, services=services, session=async_session)
await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
assert processed is True
assert job is not None
assert job.status == JobStatus.TRANSCRIBED
assert any(job_source.raw_transcription == "Pipeline transcript" for job_source in job.job_sources)
assert any(job_source.prompt_name == "transcribe_document.md" for job_source in job.job_sources)
assert any(job_source.user_prompt is not None for job_source in job.job_sources)
assert any(job_source.temperature == 0.2 for job_source in job.job_sources)
assert any(job_source.top_p == 0.85 for job_source in job.job_sources)
assert any(job_source.ai_metadata == {"finish_reason": "stop", "usage": {"total_tokens": 42}} for job_source in job.job_sources)
assert any(
job_source.raw_api_response == {"id": "resp_123", "choices": [{"message": {"content": "Pipeline transcript"}}]}
@@ -153,7 +163,7 @@ class TestPipelineSuccessFlow:
queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
assert queued_job is not None
await advance_job(job=queued_job, services=services, session=async_session)
await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
assert job.status == JobStatus.TRANSCRIBED
@@ -216,7 +226,7 @@ class TestPipelineSuccessFlow:
queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
assert queued_job is not None
await advance_job(job=queued_job, services=services, session=async_session)
await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
assert job.status == JobStatus.PARTIAL_SUCCESS
@@ -289,7 +299,7 @@ class TestPipelineSuccessFlow:
queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
assert queued_job is not None
await advance_job(job=queued_job, services=services, session=async_session)
await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
refreshed = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
assert call_count == 1
@@ -337,7 +347,7 @@ class TestPipelineFailureFlow:
queued_job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
processed = queued_job is not None
if queued_job is not None:
await advance_job(job=queued_job, services=services, session=async_session)
await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
assert processed is True
+21
View File
@@ -73,6 +73,27 @@ class TestOpenRouterProviderTranscribe:
assert send_call["x_open_router_title"] == "Transcription App"
assert result.text == "Transcript text"
@pytest.mark.asyncio
async def test_includes_temperature_and_top_p_when_provided(self):
"""Transcribe passes configured sampling parameters through to OpenRouter."""
response = {"model": "vendor/model-a", "choices": [{"message": {"content": "Transcript text"}}]}
client = _FakeClient(response=response)
provider = OpenRouterTranscriptionProvider(settings=Settings(openrouter_api_key="test-key"), client=client)
result = await provider.transcribe(
prompt_text="Prompt body",
image_bytes=b"img-bytes",
mime_type="image/png",
temperature=0.2,
top_p=0.85,
)
send_call = client.chat.calls[0]
assert send_call["temperature"] == 0.2
assert send_call["top_p"] == 0.85
assert result.temperature == 0.2
assert result.top_p == 0.85
@pytest.mark.asyncio
async def test_parses_successful_response_text(self):
"""Transcribe returns normalized text from a valid response payload."""
+2
View File
@@ -78,6 +78,8 @@ async def test_delete_document_blocks_when_dependencies_exist(default_session_fa
upload_name="001_page.png",
filename="001_page.png",
file_path="uploads/001_page.png",
file_hash="a" * 64,
file_size_bytes=1,
)
)
session.add(Job(document_id=document.id))
+15 -3
View File
@@ -80,6 +80,8 @@ class TestJobService:
upload_name="letter.jpg",
filename="stored-letter.jpg",
file_path="/uploads/stored-letter.jpg",
file_hash="a" * 64,
file_size_bytes=1,
)
session.add(source)
await session.flush()
@@ -120,7 +122,7 @@ class TestJobService:
assert next_job.id == first.id
@pytest.mark.asyncio
async def test_create_job_persists_provider_model_prompt(
async def test_create_job_persists_provider_and_model(
self,
job_service: JobService,
document_service: DocumentService,
@@ -132,14 +134,12 @@ class TestJobService:
document_id=document.id,
provider="openrouter",
model="google/gemini-2.5-flash",
prompt_name="transcribe_document.md",
)
await job_service.create_job(job=job)
fetched = await job_service.read_job(job_id=job.id)
assert fetched.provider == "openrouter"
assert fetched.model == "google/gemini-2.5-flash"
assert fetched.prompt_name == "transcribe_document.md"
@pytest.mark.asyncio
async def test_read_job_resolves_filename_from_linked_source(
@@ -160,6 +160,8 @@ class TestJobService:
upload_name="page_001.png",
filename="stored_page_001.png",
file_path="/uploads/stored_page_001.png",
file_hash="b" * 64,
file_size_bytes=1,
)
session.add(source)
await session.flush()
@@ -210,6 +212,8 @@ class TestJobService:
upload_name="delete-job-source.jpg",
filename="stored-delete-job-source.jpg",
file_path="/uploads/stored-delete-job-source.jpg",
file_hash="c" * 64,
file_size_bytes=1,
)
session.add(source)
await session.flush()
@@ -246,6 +250,8 @@ class TestJobService:
upload_name="cancel-1.jpg",
filename="stored-cancel-1.jpg",
file_path="/uploads/stored-cancel-1.jpg",
file_hash="d" * 64,
file_size_bytes=1,
)
source_two = Source(
document_id=document.id,
@@ -253,6 +259,8 @@ class TestJobService:
upload_name="cancel-2.jpg",
filename="stored-cancel-2.jpg",
file_path="/uploads/stored-cancel-2.jpg",
file_hash="e" * 64,
file_size_bytes=1,
)
session.add(source_one)
session.add(source_two)
@@ -304,6 +312,8 @@ class TestJobService:
upload_name="resubmit-1.jpg",
filename="stored-resubmit-1.jpg",
file_path="/uploads/stored-resubmit-1.jpg",
file_hash="f" * 64,
file_size_bytes=1,
raw_transcription="existing text",
)
source_two = Source(
@@ -312,6 +322,8 @@ class TestJobService:
upload_name="resubmit-2.jpg",
filename="stored-resubmit-2.jpg",
file_path="/uploads/stored-resubmit-2.jpg",
file_hash="0" * 64,
file_size_bytes=1,
raw_transcription="done text",
)
session.add(source_one)
+8 -2
View File
@@ -48,7 +48,6 @@ async def test_create_job_for_document_sorts_uploads_and_creates_links(async_ses
],
provider="openrouter",
model="test-model",
prompt_name="transcribe_document.md",
session=async_session,
settings=settings,
)
@@ -57,7 +56,6 @@ async def test_create_job_for_document_sorts_uploads_and_creates_links(async_ses
assert created_job is not None
assert created_job.provider == "openrouter"
assert created_job.model == "test-model"
assert created_job.prompt_name == "transcribe_document.md"
sources = (
await async_session.exec(
@@ -71,10 +69,16 @@ async def test_create_job_for_document_sorts_uploads_and_creates_links(async_ses
assert all("A_page" not in source.filename and "b_page" not in source.filename for source in sources)
assert all(Path(source.filename).stem == str(source.id) for source in sources)
assert all(Path(source.file_path).parent == (tmp_path / "documents" / str(document.id)) for source in sources)
assert [source.file_hash for source in sources] == [
"ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb",
"3e23e8160039594a33894f6564e1b1348bbd7a0088d42c4acb73eeaed59c009d",
]
assert [source.file_size_bytes for source in sources] == [1, 1]
job_sources = (await async_session.exec(select(JobSource).where(JobSource.job_id == result.job_id))).all()
assert len(job_sources) == 2
assert set(result.source_ids) == {job_source.source_id for job_source in job_sources}
assert {job_source.prompt_name for job_source in job_sources} == {None}
@pytest.mark.asyncio
@@ -103,6 +107,8 @@ async def test_create_upload_job_stores_source_under_document_id_directory(async
assert Path(source.filename).stem == str(source.id)
assert result.stored_path.name == source.filename
assert Path(source.file_path).parent == expected_parent
assert source.file_hash == "2c8648d103e3dd7ad87660da0f126a1443b6d21ac1bd3ec000c5e24e2373a90c"
assert source.file_size_bytes == len(b"image-bytes")
def test_store_person_portrait_stores_file_under_person_id_directory(tmp_path):
@@ -39,6 +39,8 @@ class TestTranscriptionServiceRevisionUpsert:
upload_name="source.jpg",
filename="source.jpg",
file_path="uploads/source.jpg",
file_hash="1" * 64,
file_size_bytes=1,
)
async with transcriptions._session_scope() as session:
session.add(source)
@@ -74,6 +76,8 @@ class TestTranscriptionServiceRevisionUpsert:
upload_name="source.jpg",
filename="source.jpg",
file_path="uploads/source.jpg",
file_hash="2" * 64,
file_size_bytes=1,
)
async with transcriptions._session_scope() as session:
session.add(source)
@@ -115,6 +119,8 @@ class TestTranscriptionServiceRevisionUpsert:
upload_name="delete.jpg",
filename="delete.jpg",
file_path=str(stored_path),
file_hash="3" * 64,
file_size_bytes=4,
)
async with transcriptions._session_scope() as session:
session.add(source)
@@ -149,6 +155,8 @@ class TestTranscriptionServiceRevisionUpsert:
upload_name="shared.jpg",
filename="shared.jpg",
file_path="uploads/shared.jpg",
file_hash="4" * 64,
file_size_bytes=1,
)
async with transcriptions._session_scope() as session:
session.add(source)
@@ -180,6 +188,8 @@ class TestTranscriptionServiceRevisionUpsert:
upload_name="orphan.jpg",
filename="orphan.jpg",
file_path=str(stored_path),
file_hash="5" * 64,
file_size_bytes=4,
)
await transcriptions.create_source(source=source)
@@ -207,6 +217,8 @@ class TestTranscriptionServiceRevisionUpsert:
upload_name="linked.jpg",
filename="linked.jpg",
file_path="uploads/linked.jpg",
file_hash="6" * 64,
file_size_bytes=1,
)
async with transcriptions._session_scope() as session:
session.add(source)
+11
View File
@@ -60,6 +60,8 @@ async def test_transcription_service_manages_source_crud(default_session_factory
upload_name="page-1.jpg",
filename="page-1.jpg",
file_path="uploads/page-1.jpg",
file_hash="7" * 64,
file_size_bytes=1,
)
)
@@ -101,6 +103,8 @@ async def test_transcription_service_job_source_crud_uses_caller_session(default
upload_name="job-source.jpg",
filename="job-source.jpg",
file_path="uploads/job-source.jpg",
file_hash="8" * 64,
file_size_bytes=1,
)
session.add(source)
await session.flush()
@@ -165,6 +169,8 @@ async def test_document_delete_is_blocked_with_source_and_job_dependencies(defau
upload_name="blocked.jpg",
filename="blocked.jpg",
file_path="uploads/blocked.jpg",
file_hash="9" * 64,
file_size_bytes=1,
)
)
await transcriptions.create_job_source(
@@ -200,6 +206,8 @@ async def test_source_delete_blocks_when_linked_to_multiple_jobs(default_session
upload_name="shared-page.jpg",
filename="shared-page.jpg",
file_path="uploads/shared-page.jpg",
file_hash="a" * 64,
file_size_bytes=1,
)
)
await transcriptions.create_job_source(
@@ -228,6 +236,8 @@ async def test_update_job_source_transcription_persists_provider_json_payloads(d
upload_name="provider.jpg",
filename="provider.jpg",
file_path="uploads/provider.jpg",
file_hash="b" * 64,
file_size_bytes=1,
)
)
await transcriptions.create_job_source(
@@ -251,5 +261,6 @@ async def test_update_job_source_transcription_persists_provider_json_payloads(d
stored_rows = await transcriptions.list_job_sources(job_id=job.id)
assert len(stored_rows) == 1
assert stored_rows[0].raw_transcription == "provider transcript"
assert stored_rows[0].prompt_name == "transcribe_document.md"
assert stored_rows[0].ai_metadata == metadata
assert stored_rows[0].raw_api_response == raw_payload
+7 -1
View File
@@ -8,6 +8,8 @@ import pytest
from transcription.config import Settings
from transcription.db.models import Document
from transcription.db.models import Job
from transcription.db.models import JobSource
from transcription.db.models import JobSourceStatus
from transcription.db.models import JobStatus
from transcription.db.models import Source
from transcription.services import ServiceBundle
@@ -49,12 +51,16 @@ class TestWorkflowReliability:
source = Source(
document_id=document.id,
job_id=job.id,
page_number=1,
upload_name="timeout.jpg",
filename="timeout.jpg",
file_path=str(Path("tests/fixtures/images/real/Book Two - page 02.jpg")),
file_hash="c" * 64,
file_size_bytes=1,
)
session.add(source)
await session.flush()
session.add(JobSource(job_id=job.id, source_id=source.id, status=JobSourceStatus.PENDING))
await session.commit()
loaded = await services.jobs.read_job(job_id=job.id, session=session)
+2
View File
@@ -64,6 +64,8 @@ def _persist_source(session, document: Document, *, page_number: int = 1, **over
"raw_transcription": "Original machine text",
}
defaults.update(overrides)
defaults["file_hash"] = "a" * 64
defaults["file_size_bytes"] = 123
source = Source(**defaults)
session.add(source)
session.commit()
+4 -1
View File
@@ -95,16 +95,18 @@ async def seed_job(app_client: tuple[FastAPI, TestClient]) -> Callable[..., Awai
retry_count=0,
provider="openrouter",
model="google/gemini-2.5-flash",
prompt_name="transcribe_document.md",
)
session.add(job)
await session.flush()
source = Source(
document_id=document.id,
page_number=1,
upload_name=filename,
filename=filename,
file_path=str(stored_path),
file_hash="b" * 64,
file_size_bytes=len(stored_path.read_bytes()),
)
session.add(source)
await session.flush()
@@ -119,6 +121,7 @@ async def seed_job(app_client: tuple[FastAPI, TestClient]) -> Callable[..., Awai
if transcription_text is not None
else JobSourceStatus.FAILED
),
prompt_name="transcribe_document.md",
raw_transcription=transcription_text,
error_detail=error_detail,
)
+2
View File
@@ -146,6 +146,8 @@ class TestDocumentsPageRendering:
upload_name="page_1.png",
filename="page_1.png",
file_path="/tmp/page_1.png",
file_hash="0" * 64,
file_size_bytes=1,
)
session.add(source)
await session.commit()
+10
View File
@@ -23,6 +23,8 @@ class TestSourceModelProperties:
upload_name="page_one.png",
filename="stored_page_one.png",
file_path="/tmp/stored_page_one.png",
file_hash="b" * 64,
file_size_bytes=1,
)
assert source.latest_job_source is None
@@ -86,6 +88,8 @@ class TestSourcesPageRendering:
upload_name="page_one.png",
filename="stored_page_one.png",
file_path="/tmp/stored_page_one.png",
file_hash="c" * 64,
file_size_bytes=1,
)
)
await session.commit()
@@ -114,6 +118,8 @@ class TestSourcesPageRendering:
upload_name="target_page.png",
filename="target_stored.png",
file_path="/tmp/target_stored.png",
file_hash="d" * 64,
file_size_bytes=1,
),
Source(
document_id=other.id,
@@ -121,6 +127,8 @@ class TestSourcesPageRendering:
upload_name="other_page.png",
filename="other_stored.png",
file_path="/tmp/other_stored.png",
file_hash="e" * 64,
file_size_bytes=1,
),
]
)
@@ -240,6 +248,8 @@ class TestSourcesPageRendering:
upload_name="orphan-source.png",
filename="orphan-source.png",
file_path="/tmp/orphan-source.png",
file_hash="f" * 64,
file_size_bytes=1,
)
session.add(source)
await session.commit()