V4.2 Updated what ai_raw_response data is being captured. The changes were more extensive than I expected.

tests/services/test_workflows_reliability.py

@@ -1,5 +1,6 @@
 """Reliability tests for worker workflow timeout behavior."""
 
+import asyncio
 from pathlib import Path
 from uuid import uuid4
 
@@ -12,6 +13,7 @@ from transcription.db.models import JobSource
 from transcription.db.models import JobSourceStatus
 from transcription.db.models import JobStatus
 from transcription.db.models import Source
+from transcription.providers.base import TranscriptionResult
 from transcription.services import ServiceBundle
 from transcription.services.workflows import process_queued_job
 
@@ -88,3 +90,64 @@ class TestWorkflowReliability:
         assert result.error_detail is not None
         assert "timed out" in result.error_detail.lower()
         assert "20.0s" in result.error_detail
+
+    @pytest.mark.asyncio
+    async def test_completed_page_is_committed_before_next_provider_call_finishes(
+        self,
+        default_session_factory,
+        monkeypatch,
+    ):
+        services = ServiceBundle(
+            documents=ServiceBundle().documents.__class__(session_factory=default_session_factory),
+            jobs=ServiceBundle().jobs.__class__(session_factory=default_session_factory),
+            sources=ServiceBundle().sources.__class__(session_factory=default_session_factory),
+            people=ServiceBundle().people.__class__(session_factory=default_session_factory),
+        )
+        async with services.jobs._session_scope() as session:
+            document = Document(id=uuid4(), name="durability-doc")
+            session.add(document)
+            await session.flush()
+            job = Job(document_id=document.id, status=JobStatus.QUEUED)
+            session.add(job)
+            await session.flush()
+            for page_number in (1, 2):
+                source = Source(
+                    document_id=document.id,
+                    page_number=page_number,
+                    upload_name=f"page-{page_number}.jpg",
+                    filename=f"page-{page_number}.jpg",
+                    file_path=str(Path("tests/fixtures/images/real/Book Two - page 02.jpg")),
+                    file_hash=str(page_number) * 64,
+                    file_size_bytes=1,
+                )
+                session.add(source)
+                await session.flush()
+                session.add(JobSource(job_id=job.id, source_id=source.id))
+            await session.commit()
+            loaded = await services.jobs.read_job(job_id=job.id, session=session)
+
+        second_started = asyncio.Event()
+        release_second = asyncio.Event()
+        call_count = 0
+
+        async def _transcribe(image_path, **kwargs):
+            nonlocal call_count
+            _ = (image_path, kwargs)
+            call_count += 1
+            if call_count == 2:
+                second_started.set()
+                await release_second.wait()
+            return TranscriptionResult(text=f"page {call_count}", provider="fixture", model="model")
+
+        monkeypatch.setattr("transcription.services.workflows.transcribe_document_image", _transcribe)
+        task = asyncio.create_task(process_queued_job(job=loaded, services=services))
+        await asyncio.wait_for(second_started.wait(), timeout=2)
+
+        attempts = await services.sources.list_execution_attempts(job_id=job.id)
+        assert len(attempts) == 1
+        assert attempts[0].raw_transcription == "page 1"
+
+        release_second.set()
+        result = await task
+        assert result is not None
+        assert result.status == JobStatus.TRANSCRIBED