generated from john/python-template
V3 step 1 update models.py and step 2 implement service/worker, and raw API response persistence
This commit is contained in:
@@ -46,7 +46,13 @@ class TestPipelineSuccessFlow:
|
||||
self, async_session, default_session_factory, tmp_path: Path, monkeypatch
|
||||
):
|
||||
"""Upload followed by worker processing persists job transcription and transcribed status."""
|
||||
settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
|
||||
settings = Settings(
|
||||
openrouter_api_key="test-key",
|
||||
upload_dir=tmp_path,
|
||||
default_prompt_name="transcribe_document.md",
|
||||
transcription_temperature=0.2,
|
||||
transcription_top_p=0.85,
|
||||
)
|
||||
upload_result = await create_upload_job(
|
||||
filename="pipeline.jpg",
|
||||
file_bytes=b"pipeline-bytes",
|
||||
@@ -89,13 +95,17 @@ class TestPipelineSuccessFlow:
|
||||
queued_job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
|
||||
processed = queued_job is not None
|
||||
if queued_job is not None:
|
||||
await advance_job(job=queued_job, services=services, session=async_session)
|
||||
await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
|
||||
job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
|
||||
|
||||
assert processed is True
|
||||
assert job is not None
|
||||
assert job.status == JobStatus.TRANSCRIBED
|
||||
assert any(job_source.raw_transcription == "Pipeline transcript" for job_source in job.job_sources)
|
||||
assert any(job_source.prompt_name == "transcribe_document.md" for job_source in job.job_sources)
|
||||
assert any(job_source.user_prompt is not None for job_source in job.job_sources)
|
||||
assert any(job_source.temperature == 0.2 for job_source in job.job_sources)
|
||||
assert any(job_source.top_p == 0.85 for job_source in job.job_sources)
|
||||
assert any(job_source.ai_metadata == {"finish_reason": "stop", "usage": {"total_tokens": 42}} for job_source in job.job_sources)
|
||||
assert any(
|
||||
job_source.raw_api_response == {"id": "resp_123", "choices": [{"message": {"content": "Pipeline transcript"}}]}
|
||||
@@ -153,7 +163,7 @@ class TestPipelineSuccessFlow:
|
||||
queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
|
||||
assert queued_job is not None
|
||||
|
||||
await advance_job(job=queued_job, services=services, session=async_session)
|
||||
await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
|
||||
job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
|
||||
|
||||
assert job.status == JobStatus.TRANSCRIBED
|
||||
@@ -216,7 +226,7 @@ class TestPipelineSuccessFlow:
|
||||
queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
|
||||
assert queued_job is not None
|
||||
|
||||
await advance_job(job=queued_job, services=services, session=async_session)
|
||||
await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
|
||||
job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
|
||||
|
||||
assert job.status == JobStatus.PARTIAL_SUCCESS
|
||||
@@ -289,7 +299,7 @@ class TestPipelineSuccessFlow:
|
||||
|
||||
queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
|
||||
assert queued_job is not None
|
||||
await advance_job(job=queued_job, services=services, session=async_session)
|
||||
await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
|
||||
|
||||
refreshed = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
|
||||
assert call_count == 1
|
||||
@@ -337,7 +347,7 @@ class TestPipelineFailureFlow:
|
||||
queued_job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
|
||||
processed = queued_job is not None
|
||||
if queued_job is not None:
|
||||
await advance_job(job=queued_job, services=services, session=async_session)
|
||||
await advance_job(job=queued_job, services=services, settings=settings, session=async_session)
|
||||
job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
|
||||
|
||||
assert processed is True
|
||||
|
||||
@@ -73,6 +73,27 @@ class TestOpenRouterProviderTranscribe:
|
||||
assert send_call["x_open_router_title"] == "Transcription App"
|
||||
assert result.text == "Transcript text"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_includes_temperature_and_top_p_when_provided(self):
|
||||
"""Transcribe passes configured sampling parameters through to OpenRouter."""
|
||||
response = {"model": "vendor/model-a", "choices": [{"message": {"content": "Transcript text"}}]}
|
||||
client = _FakeClient(response=response)
|
||||
provider = OpenRouterTranscriptionProvider(settings=Settings(openrouter_api_key="test-key"), client=client)
|
||||
|
||||
result = await provider.transcribe(
|
||||
prompt_text="Prompt body",
|
||||
image_bytes=b"img-bytes",
|
||||
mime_type="image/png",
|
||||
temperature=0.2,
|
||||
top_p=0.85,
|
||||
)
|
||||
|
||||
send_call = client.chat.calls[0]
|
||||
assert send_call["temperature"] == 0.2
|
||||
assert send_call["top_p"] == 0.85
|
||||
assert result.temperature == 0.2
|
||||
assert result.top_p == 0.85
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parses_successful_response_text(self):
|
||||
"""Transcribe returns normalized text from a valid response payload."""
|
||||
|
||||
@@ -78,6 +78,8 @@ async def test_delete_document_blocks_when_dependencies_exist(default_session_fa
|
||||
upload_name="001_page.png",
|
||||
filename="001_page.png",
|
||||
file_path="uploads/001_page.png",
|
||||
file_hash="a" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
)
|
||||
session.add(Job(document_id=document.id))
|
||||
|
||||
@@ -80,6 +80,8 @@ class TestJobService:
|
||||
upload_name="letter.jpg",
|
||||
filename="stored-letter.jpg",
|
||||
file_path="/uploads/stored-letter.jpg",
|
||||
file_hash="a" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
session.add(source)
|
||||
await session.flush()
|
||||
@@ -120,7 +122,7 @@ class TestJobService:
|
||||
assert next_job.id == first.id
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_job_persists_provider_model_prompt(
|
||||
async def test_create_job_persists_provider_and_model(
|
||||
self,
|
||||
job_service: JobService,
|
||||
document_service: DocumentService,
|
||||
@@ -132,14 +134,12 @@ class TestJobService:
|
||||
document_id=document.id,
|
||||
provider="openrouter",
|
||||
model="google/gemini-2.5-flash",
|
||||
prompt_name="transcribe_document.md",
|
||||
)
|
||||
await job_service.create_job(job=job)
|
||||
|
||||
fetched = await job_service.read_job(job_id=job.id)
|
||||
assert fetched.provider == "openrouter"
|
||||
assert fetched.model == "google/gemini-2.5-flash"
|
||||
assert fetched.prompt_name == "transcribe_document.md"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_job_resolves_filename_from_linked_source(
|
||||
@@ -160,6 +160,8 @@ class TestJobService:
|
||||
upload_name="page_001.png",
|
||||
filename="stored_page_001.png",
|
||||
file_path="/uploads/stored_page_001.png",
|
||||
file_hash="b" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
session.add(source)
|
||||
await session.flush()
|
||||
@@ -210,6 +212,8 @@ class TestJobService:
|
||||
upload_name="delete-job-source.jpg",
|
||||
filename="stored-delete-job-source.jpg",
|
||||
file_path="/uploads/stored-delete-job-source.jpg",
|
||||
file_hash="c" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
session.add(source)
|
||||
await session.flush()
|
||||
@@ -246,6 +250,8 @@ class TestJobService:
|
||||
upload_name="cancel-1.jpg",
|
||||
filename="stored-cancel-1.jpg",
|
||||
file_path="/uploads/stored-cancel-1.jpg",
|
||||
file_hash="d" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
source_two = Source(
|
||||
document_id=document.id,
|
||||
@@ -253,6 +259,8 @@ class TestJobService:
|
||||
upload_name="cancel-2.jpg",
|
||||
filename="stored-cancel-2.jpg",
|
||||
file_path="/uploads/stored-cancel-2.jpg",
|
||||
file_hash="e" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
session.add(source_one)
|
||||
session.add(source_two)
|
||||
@@ -304,6 +312,8 @@ class TestJobService:
|
||||
upload_name="resubmit-1.jpg",
|
||||
filename="stored-resubmit-1.jpg",
|
||||
file_path="/uploads/stored-resubmit-1.jpg",
|
||||
file_hash="f" * 64,
|
||||
file_size_bytes=1,
|
||||
raw_transcription="existing text",
|
||||
)
|
||||
source_two = Source(
|
||||
@@ -312,6 +322,8 @@ class TestJobService:
|
||||
upload_name="resubmit-2.jpg",
|
||||
filename="stored-resubmit-2.jpg",
|
||||
file_path="/uploads/stored-resubmit-2.jpg",
|
||||
file_hash="0" * 64,
|
||||
file_size_bytes=1,
|
||||
raw_transcription="done text",
|
||||
)
|
||||
session.add(source_one)
|
||||
|
||||
@@ -48,7 +48,6 @@ async def test_create_job_for_document_sorts_uploads_and_creates_links(async_ses
|
||||
],
|
||||
provider="openrouter",
|
||||
model="test-model",
|
||||
prompt_name="transcribe_document.md",
|
||||
session=async_session,
|
||||
settings=settings,
|
||||
)
|
||||
@@ -57,7 +56,6 @@ async def test_create_job_for_document_sorts_uploads_and_creates_links(async_ses
|
||||
assert created_job is not None
|
||||
assert created_job.provider == "openrouter"
|
||||
assert created_job.model == "test-model"
|
||||
assert created_job.prompt_name == "transcribe_document.md"
|
||||
|
||||
sources = (
|
||||
await async_session.exec(
|
||||
@@ -71,10 +69,16 @@ async def test_create_job_for_document_sorts_uploads_and_creates_links(async_ses
|
||||
assert all("A_page" not in source.filename and "b_page" not in source.filename for source in sources)
|
||||
assert all(Path(source.filename).stem == str(source.id) for source in sources)
|
||||
assert all(Path(source.file_path).parent == (tmp_path / "documents" / str(document.id)) for source in sources)
|
||||
assert [source.file_hash for source in sources] == [
|
||||
"ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb",
|
||||
"3e23e8160039594a33894f6564e1b1348bbd7a0088d42c4acb73eeaed59c009d",
|
||||
]
|
||||
assert [source.file_size_bytes for source in sources] == [1, 1]
|
||||
|
||||
job_sources = (await async_session.exec(select(JobSource).where(JobSource.job_id == result.job_id))).all()
|
||||
assert len(job_sources) == 2
|
||||
assert set(result.source_ids) == {job_source.source_id for job_source in job_sources}
|
||||
assert {job_source.prompt_name for job_source in job_sources} == {None}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -103,6 +107,8 @@ async def test_create_upload_job_stores_source_under_document_id_directory(async
|
||||
assert Path(source.filename).stem == str(source.id)
|
||||
assert result.stored_path.name == source.filename
|
||||
assert Path(source.file_path).parent == expected_parent
|
||||
assert source.file_hash == "2c8648d103e3dd7ad87660da0f126a1443b6d21ac1bd3ec000c5e24e2373a90c"
|
||||
assert source.file_size_bytes == len(b"image-bytes")
|
||||
|
||||
|
||||
def test_store_person_portrait_stores_file_under_person_id_directory(tmp_path):
|
||||
|
||||
@@ -39,6 +39,8 @@ class TestTranscriptionServiceRevisionUpsert:
|
||||
upload_name="source.jpg",
|
||||
filename="source.jpg",
|
||||
file_path="uploads/source.jpg",
|
||||
file_hash="1" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
async with transcriptions._session_scope() as session:
|
||||
session.add(source)
|
||||
@@ -74,6 +76,8 @@ class TestTranscriptionServiceRevisionUpsert:
|
||||
upload_name="source.jpg",
|
||||
filename="source.jpg",
|
||||
file_path="uploads/source.jpg",
|
||||
file_hash="2" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
async with transcriptions._session_scope() as session:
|
||||
session.add(source)
|
||||
@@ -115,6 +119,8 @@ class TestTranscriptionServiceRevisionUpsert:
|
||||
upload_name="delete.jpg",
|
||||
filename="delete.jpg",
|
||||
file_path=str(stored_path),
|
||||
file_hash="3" * 64,
|
||||
file_size_bytes=4,
|
||||
)
|
||||
async with transcriptions._session_scope() as session:
|
||||
session.add(source)
|
||||
@@ -149,6 +155,8 @@ class TestTranscriptionServiceRevisionUpsert:
|
||||
upload_name="shared.jpg",
|
||||
filename="shared.jpg",
|
||||
file_path="uploads/shared.jpg",
|
||||
file_hash="4" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
async with transcriptions._session_scope() as session:
|
||||
session.add(source)
|
||||
@@ -180,6 +188,8 @@ class TestTranscriptionServiceRevisionUpsert:
|
||||
upload_name="orphan.jpg",
|
||||
filename="orphan.jpg",
|
||||
file_path=str(stored_path),
|
||||
file_hash="5" * 64,
|
||||
file_size_bytes=4,
|
||||
)
|
||||
await transcriptions.create_source(source=source)
|
||||
|
||||
@@ -207,6 +217,8 @@ class TestTranscriptionServiceRevisionUpsert:
|
||||
upload_name="linked.jpg",
|
||||
filename="linked.jpg",
|
||||
file_path="uploads/linked.jpg",
|
||||
file_hash="6" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
async with transcriptions._session_scope() as session:
|
||||
session.add(source)
|
||||
|
||||
@@ -60,6 +60,8 @@ async def test_transcription_service_manages_source_crud(default_session_factory
|
||||
upload_name="page-1.jpg",
|
||||
filename="page-1.jpg",
|
||||
file_path="uploads/page-1.jpg",
|
||||
file_hash="7" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -101,6 +103,8 @@ async def test_transcription_service_job_source_crud_uses_caller_session(default
|
||||
upload_name="job-source.jpg",
|
||||
filename="job-source.jpg",
|
||||
file_path="uploads/job-source.jpg",
|
||||
file_hash="8" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
session.add(source)
|
||||
await session.flush()
|
||||
@@ -165,6 +169,8 @@ async def test_document_delete_is_blocked_with_source_and_job_dependencies(defau
|
||||
upload_name="blocked.jpg",
|
||||
filename="blocked.jpg",
|
||||
file_path="uploads/blocked.jpg",
|
||||
file_hash="9" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
)
|
||||
await transcriptions.create_job_source(
|
||||
@@ -200,6 +206,8 @@ async def test_source_delete_blocks_when_linked_to_multiple_jobs(default_session
|
||||
upload_name="shared-page.jpg",
|
||||
filename="shared-page.jpg",
|
||||
file_path="uploads/shared-page.jpg",
|
||||
file_hash="a" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
)
|
||||
await transcriptions.create_job_source(
|
||||
@@ -228,6 +236,8 @@ async def test_update_job_source_transcription_persists_provider_json_payloads(d
|
||||
upload_name="provider.jpg",
|
||||
filename="provider.jpg",
|
||||
file_path="uploads/provider.jpg",
|
||||
file_hash="b" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
)
|
||||
await transcriptions.create_job_source(
|
||||
@@ -251,5 +261,6 @@ async def test_update_job_source_transcription_persists_provider_json_payloads(d
|
||||
stored_rows = await transcriptions.list_job_sources(job_id=job.id)
|
||||
assert len(stored_rows) == 1
|
||||
assert stored_rows[0].raw_transcription == "provider transcript"
|
||||
assert stored_rows[0].prompt_name == "transcribe_document.md"
|
||||
assert stored_rows[0].ai_metadata == metadata
|
||||
assert stored_rows[0].raw_api_response == raw_payload
|
||||
|
||||
@@ -8,6 +8,8 @@ import pytest
|
||||
from transcription.config import Settings
|
||||
from transcription.db.models import Document
|
||||
from transcription.db.models import Job
|
||||
from transcription.db.models import JobSource
|
||||
from transcription.db.models import JobSourceStatus
|
||||
from transcription.db.models import JobStatus
|
||||
from transcription.db.models import Source
|
||||
from transcription.services import ServiceBundle
|
||||
@@ -49,12 +51,16 @@ class TestWorkflowReliability:
|
||||
|
||||
source = Source(
|
||||
document_id=document.id,
|
||||
job_id=job.id,
|
||||
page_number=1,
|
||||
upload_name="timeout.jpg",
|
||||
filename="timeout.jpg",
|
||||
file_path=str(Path("tests/fixtures/images/real/Book Two - page 02.jpg")),
|
||||
file_hash="c" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
session.add(source)
|
||||
await session.flush()
|
||||
session.add(JobSource(job_id=job.id, source_id=source.id, status=JobSourceStatus.PENDING))
|
||||
await session.commit()
|
||||
|
||||
loaded = await services.jobs.read_job(job_id=job.id, session=session)
|
||||
|
||||
@@ -64,6 +64,8 @@ def _persist_source(session, document: Document, *, page_number: int = 1, **over
|
||||
"raw_transcription": "Original machine text",
|
||||
}
|
||||
defaults.update(overrides)
|
||||
defaults["file_hash"] = "a" * 64
|
||||
defaults["file_size_bytes"] = 123
|
||||
source = Source(**defaults)
|
||||
session.add(source)
|
||||
session.commit()
|
||||
|
||||
@@ -95,16 +95,18 @@ async def seed_job(app_client: tuple[FastAPI, TestClient]) -> Callable[..., Awai
|
||||
retry_count=0,
|
||||
provider="openrouter",
|
||||
model="google/gemini-2.5-flash",
|
||||
prompt_name="transcribe_document.md",
|
||||
)
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
|
||||
source = Source(
|
||||
document_id=document.id,
|
||||
page_number=1,
|
||||
upload_name=filename,
|
||||
filename=filename,
|
||||
file_path=str(stored_path),
|
||||
file_hash="b" * 64,
|
||||
file_size_bytes=len(stored_path.read_bytes()),
|
||||
)
|
||||
session.add(source)
|
||||
await session.flush()
|
||||
@@ -119,6 +121,7 @@ async def seed_job(app_client: tuple[FastAPI, TestClient]) -> Callable[..., Awai
|
||||
if transcription_text is not None
|
||||
else JobSourceStatus.FAILED
|
||||
),
|
||||
prompt_name="transcribe_document.md",
|
||||
raw_transcription=transcription_text,
|
||||
error_detail=error_detail,
|
||||
)
|
||||
|
||||
@@ -146,6 +146,8 @@ class TestDocumentsPageRendering:
|
||||
upload_name="page_1.png",
|
||||
filename="page_1.png",
|
||||
file_path="/tmp/page_1.png",
|
||||
file_hash="0" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
session.add(source)
|
||||
await session.commit()
|
||||
|
||||
@@ -23,6 +23,8 @@ class TestSourceModelProperties:
|
||||
upload_name="page_one.png",
|
||||
filename="stored_page_one.png",
|
||||
file_path="/tmp/stored_page_one.png",
|
||||
file_hash="b" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
|
||||
assert source.latest_job_source is None
|
||||
@@ -86,6 +88,8 @@ class TestSourcesPageRendering:
|
||||
upload_name="page_one.png",
|
||||
filename="stored_page_one.png",
|
||||
file_path="/tmp/stored_page_one.png",
|
||||
file_hash="c" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
@@ -114,6 +118,8 @@ class TestSourcesPageRendering:
|
||||
upload_name="target_page.png",
|
||||
filename="target_stored.png",
|
||||
file_path="/tmp/target_stored.png",
|
||||
file_hash="d" * 64,
|
||||
file_size_bytes=1,
|
||||
),
|
||||
Source(
|
||||
document_id=other.id,
|
||||
@@ -121,6 +127,8 @@ class TestSourcesPageRendering:
|
||||
upload_name="other_page.png",
|
||||
filename="other_stored.png",
|
||||
file_path="/tmp/other_stored.png",
|
||||
file_hash="e" * 64,
|
||||
file_size_bytes=1,
|
||||
),
|
||||
]
|
||||
)
|
||||
@@ -240,6 +248,8 @@ class TestSourcesPageRendering:
|
||||
upload_name="orphan-source.png",
|
||||
filename="orphan-source.png",
|
||||
file_path="/tmp/orphan-source.png",
|
||||
file_hash="f" * 64,
|
||||
file_size_bytes=1,
|
||||
)
|
||||
session.add(source)
|
||||
await session.commit()
|
||||
|
||||
Reference in New Issue
Block a user