Implement Rec - Phase 3 complete

tests/integration/test_pipeline_flow.py

@@ -99,6 +99,7 @@ class TestPipelineSuccessFlow:
             provider=None,
             source_reference=None,
             requested_model=None,
+            evidence_capture=None,
         ) -> TranscriptionResult:
             _ = (
                 image_path,
@@ -110,6 +111,7 @@ class TestPipelineSuccessFlow:
                 provider,
                 source_reference,
                 requested_model,
+                evidence_capture,
             )
             return TranscriptionResult(
                 text="Pipeline transcript",
@@ -191,9 +193,20 @@ class TestPipelineSuccessFlow:
             provider=None,
             source_reference=None,
             requested_model=None,
+            evidence_capture=None,
         ) -> TranscriptionResult:
             page_name = Path(image_path).name
-            _ = (prompt_name, prompt_text, temperature, top_p, settings, provider, source_reference, requested_model)
+            _ = (
+                prompt_name,
+                prompt_text,
+                temperature,
+                top_p,
+                settings,
+                provider,
+                source_reference,
+                requested_model,
+                evidence_capture,
+            )
             return TranscriptionResult(
                 text=f"Transcript for {page_name}",
                 provider="openrouter",
@@ -260,6 +273,7 @@ class TestPipelineSuccessFlow:
             provider=None,
             source_reference=None,
             requested_model=None,
+            evidence_capture=None,
         ) -> TranscriptionResult:
             nonlocal call_count
             call_count += 1
@@ -273,6 +287,7 @@ class TestPipelineSuccessFlow:
                 provider,
                 source_reference,
                 requested_model,
+                evidence_capture,
             )
             if call_count == 2:
                 raise RuntimeError("simulated page failure")
@@ -351,6 +366,7 @@ class TestPipelineSuccessFlow:
             provider=None,
             source_reference=None,
             requested_model=None,
+            evidence_capture=None,
         ) -> TranscriptionResult:
             nonlocal call_count
             _ = (
@@ -363,6 +379,7 @@ class TestPipelineSuccessFlow:
                 provider,
                 source_reference,
                 requested_model,
+                evidence_capture,
             )
             call_count += 1
             return TranscriptionResult(
@@ -419,6 +436,7 @@ class TestPipelineFailureFlow:
             provider=None,
             source_reference=None,
             requested_model=None,
+            evidence_capture=None,
         ) -> TranscriptionResult:
             _ = (
                 image_path,
@@ -430,6 +448,7 @@ class TestPipelineFailureFlow:
                 provider,
                 source_reference,
                 requested_model,
+                evidence_capture,
             )
             raise RuntimeError("pipeline provider failure")
 

tests/providers/test_openrouter.py

@@ -1,16 +1,21 @@
 """Tests for transcription.providers.openrouter."""
 
 import asyncio
+import json
 import logging
 from types import SimpleNamespace
 from typing import cast
+from uuid import uuid4
 
+import httpx
 import pytest
 from openrouter import OpenRouter
 
 from transcription.config import Settings
+from transcription.providers.base import ProviderCallEvidence
 from transcription.providers.base import ProviderError
 from transcription.providers.base import ProviderResponseError
+from transcription.providers.evidence import SourceEvidenceReference
 from transcription.providers.openrouter import DEFAULT_OPENROUTER_MODEL
 from transcription.providers.openrouter import OpenRouterTranscriptionProvider
 
@@ -245,3 +250,76 @@ class TestOpenRouterProviderTranscribe:
 
         assert result.request_manifest is None
         assert "request manifest omitted" in caplog.text.lower()
+
+    @pytest.mark.asyncio
+    async def test_concurrent_calls_keep_evidence_scoped_to_their_own_capture(self):
+        first_release = asyncio.Event()
+
+        async def handler(request: httpx.Request) -> httpx.Response:
+            payload = json.loads(request.content.decode("utf-8"))
+            prompt = payload["messages"][0]["content"][0]["text"]
+            if prompt == "First prompt":
+                await first_release.wait()
+                body = b'{"error":{"message":"first failure"}}'
+            else:
+                body = b'{"error":{"message":"second failure"}}'
+            return httpx.Response(
+                500,
+                content=body,
+                headers={"Content-Type": "application/json"},
+                request=request,
+            )
+
+        provider = OpenRouterTranscriptionProvider(
+            settings=Settings(openrouter_api_key="test-key"),
+            async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
+        )
+        first_capture = ProviderCallEvidence()
+        second_capture = ProviderCallEvidence()
+
+        first_task = asyncio.create_task(
+            provider.transcribe(
+                prompt_text="First prompt",
+                image_bytes=b"one",
+                mime_type="image/png",
+                evidence_capture=first_capture,
+                source_reference=SourceEvidenceReference(
+                    source_id=uuid4(),
+                    digest_sha256="1" * 64,
+                    byte_size=3,
+                    media_type="image/png",
+                    page_number=1,
+                ),
+            )
+        )
+        await asyncio.sleep(0)
+        second_task = asyncio.create_task(
+            provider.transcribe(
+                prompt_text="Second prompt",
+                image_bytes=b"two",
+                mime_type="image/png",
+                evidence_capture=second_capture,
+                source_reference=SourceEvidenceReference(
+                    source_id=uuid4(),
+                    digest_sha256="2" * 64,
+                    byte_size=3,
+                    media_type="image/png",
+                    page_number=2,
+                ),
+            )
+        )
+
+        with pytest.raises(ProviderError):
+            await second_task
+        first_release.set()
+        with pytest.raises(ProviderError):
+            await first_task
+
+        assert first_capture.request_manifest is not None
+        assert first_capture.request_manifest.prompt_content == "First prompt"
+        assert first_capture.transport_evidence is not None
+        assert first_capture.transport_evidence.body == b'{"error":{"message":"first failure"}}'
+        assert second_capture.request_manifest is not None
+        assert second_capture.request_manifest.prompt_content == "Second prompt"
+        assert second_capture.transport_evidence is not None
+        assert second_capture.transport_evidence.body == b'{"error":{"message":"second failure"}}'

tests/services/test_workflows_reliability.py

@@ -86,6 +86,7 @@ class TestWorkflowReliability:
             provider=None,
             source_reference=None,
             requested_model=None,
+            evidence_capture=None,
         ):
             _ = (
                 image_path,
@@ -97,6 +98,7 @@ class TestWorkflowReliability:
                 provider,
                 source_reference,
                 requested_model,
+                evidence_capture,
             )
             raise TimeoutError("simulated provider timeout")
 
@@ -370,6 +372,64 @@ class TestWorkflowReliability:
         assert result is not None
         assert result.status == JobStatus.TRANSCRIBED
 
+    @pytest.mark.asyncio
+    async def test_intermediate_page_commit_advances_job_liveness_timestamp(
+        self,
+        default_session_factory,
+        monkeypatch,
+    ):
+        services = ServiceBundle.from_session_factory(default_session_factory)
+        async with services.jobs._session_scope() as session:
+            document = Document(id=uuid4(), name="heartbeat-doc")
+            session.add(document)
+            await session.flush()
+            job = Job(document_id=document.id, status=JobStatus.QUEUED)
+            session.add(job)
+            await session.flush()
+            for page_number in (1, 2, 3):
+                source = Source(
+                    document_id=document.id,
+                    page_number=page_number,
+                    upload_name=f"page-{page_number}.jpg",
+                    filename=f"page-{page_number}.jpg",
+                    file_path=str(Path("tests/fixtures/images/real/Book Two - page 02.jpg")),
+                    file_hash=str(page_number) * 64,
+                    file_size_bytes=1,
+                )
+                session.add(source)
+                await session.flush()
+                session.add(JobSource(job_id=job.id, source_id=source.id))
+            await session.commit()
+            loaded = await services.jobs.read_job(job_id=job.id, session=session)
+
+        third_started = asyncio.Event()
+        release_third = asyncio.Event()
+        call_count = 0
+
+        async def _transcribe(image_path, **kwargs):
+            nonlocal call_count
+            _ = (image_path, kwargs)
+            call_count += 1
+            if call_count == 3:
+                third_started.set()
+                await release_third.wait()
+            return TranscriptionResult(text=f"page {call_count}", provider="fixture", model="model")
+
+        monkeypatch.setattr("transcription.services.workflows.transcribe_document_image", _transcribe)
+        task = asyncio.create_task(process_queued_job(job=loaded, services=services))
+        await asyncio.wait_for(third_started.wait(), timeout=2)
+
+        async with services.jobs._session_scope() as session:
+            current_job = await session.get(Job, loaded.id)
+            assert current_job is not None
+            attempts = await services.evidence.list_execution_attempts(job_id=loaded.id, session=session)
+
+        assert len(attempts) == 2
+        assert current_job.date_updated >= attempts[1].finished_at
+
+        release_third.set()
+        await task
+
     @pytest.mark.asyncio
     async def test_failed_job_with_validation_category_is_not_requeued(self, default_session_factory):
         services = ServiceBundle.from_session_factory(default_session_factory)

tests/test_config.py

@@ -167,11 +167,29 @@ class TestWorkerReliabilitySettings:
         settings = _make_settings()
         assert settings.run_embedded_worker is True
         assert settings.worker_max_retries == 0
-        assert settings.worker_stale_job_seconds == 30.0
+        assert settings.worker_stale_job_seconds == 90.0
         assert settings.worker_retry_backoff_seconds == 1.0
         assert settings.worker_shutdown_grace_seconds == 5.0
         assert settings.worker_poll_interval_seconds == 1.0
 
+    def test_worker_stale_threshold_defaults_to_three_times_provider_timeout(self):
+        settings = _make_settings(worker_provider_timeout_seconds=45.0)
+
+        assert settings.worker_stale_job_seconds == 135.0
+
+    def test_worker_stale_threshold_derives_from_environment_timeout(self, monkeypatch):
+        monkeypatch.setenv("OPENROUTER_API_KEY", "test-key")
+        monkeypatch.setenv("WORKER_PROVIDER_TIMEOUT_SECONDS", "45.0")
+        monkeypatch.delenv("WORKER_STALE_JOB_SECONDS", raising=False)
+
+        settings = Settings(_env_file=None)
+
+        assert settings.worker_stale_job_seconds == 135.0
+
+    def test_worker_stale_threshold_must_exceed_provider_timeout(self):
+        with pytest.raises(ValidationError):
+            _make_settings(worker_provider_timeout_seconds=45.0, worker_stale_job_seconds=45.0)
+
 
 def test_provider_timeout_is_not_capped_at_twenty_seconds():
     """HIGH-03: vision transcription regularly runs past the old le=20.0 ceiling."""