"""Focused V4.2 evidence, integrity, and benchmark tests.""" from __future__ import annotations import hashlib import json from datetime import UTC from datetime import datetime from uuid import uuid4 import httpx import pytest from transcription.benchmarking import EditorialAssessment from transcription.benchmarking import score_transcription from transcription.config import Settings from transcription.db.models import Document from transcription.db.models import Job from transcription.db.models import JobSource from transcription.db.models import JobSourceStatus from transcription.db.models import ProcessingArtifact from transcription.db.models import Source from transcription.providers.base import ProviderError from transcription.providers.base import TranscriptionResult from transcription.providers.evidence import SourceEvidenceReference from transcription.providers.openrouter import OpenRouterTranscriptionProvider from transcription.services.documents import DocumentService from transcription.services.jobs import JobDeleteBlockedError from transcription.services.jobs import JobService from transcription.services.sources import SourceDeleteBlockedError from transcription.services.sources import SourceService from transcription.services.sources import TranscriptionError from transcription.services.sources import transcribe_document_image @pytest.mark.asyncio async def test_openrouter_captures_exact_transport_and_secret_safe_manifest(): response_body = ( b'{"id":"gen-1","created":1,"model":"vendor/model","object":"chat.completion",' b'"system_fingerprint":null,"choices":[{"index":0,"finish_reason":"stop",' b'"message":{"role":"assistant","content":"Transcript"}}],' b'"unknown_transport_field":{"retained":true}}' ) async def handler(request: httpx.Request) -> httpx.Response: assert request.headers["authorization"] == "Bearer test-key" return httpx.Response( 200, content=response_body, headers={ "Content-Type": "application/json", "X-Request-Id": "req-123", "Set-Cookie": "must-not-persist=1", }, request=request, ) async_client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) provider = OpenRouterTranscriptionProvider( settings=Settings(openrouter_api_key="test-key"), async_client=async_client, ) result = await provider.transcribe( prompt_text="Literal prompt", image_bytes=b"source-bytes", mime_type="image/png", source_reference=SourceEvidenceReference( source_id=uuid4(), digest_sha256=hashlib.sha256(b"source-bytes").hexdigest(), byte_size=len(b"source-bytes"), media_type="image/png", page_number=1, ), ) assert result.transport_evidence is not None assert result.transport_evidence.body == response_body assert result.transport_evidence.safe_headers == { "content-type": "application/json", "x-request-id": "req-123", } assert b'"unknown_transport_field":{"retained":true}' in result.transport_evidence.body assert result.request_manifest is not None serialized_manifest = json.dumps(result.request_manifest.model_dump(mode="json")) assert "data:image/png;base64" not in serialized_manifest assert "test-key" not in serialized_manifest assert result.request_manifest.omitted_optional_parameters == ("temperature", "top_p") @pytest.mark.asyncio async def test_openrouter_failure_retains_safe_response_evidence(): async def handler(request: httpx.Request) -> httpx.Response: return httpx.Response( 500, content=b'{"error":{"message":"provider unavailable"}}', headers={"Content-Type": "application/json", "Retry-After": "2", "Set-Cookie": "secret=1"}, request=request, ) provider = OpenRouterTranscriptionProvider( settings=Settings(openrouter_api_key="test-key"), async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), ) with pytest.raises(ProviderError) as failure: await provider.transcribe( prompt_text="Literal prompt", image_bytes=b"source-bytes", mime_type="image/png", ) evidence = failure.value.transport_evidence assert evidence is not None assert evidence.status_code == 500 assert evidence.body == b'{"error":{"message":"provider unavailable"}}' assert evidence.safe_headers == {"content-type": "application/json", "retry-after": "2"} @pytest.mark.asyncio async def test_openrouter_does_not_reuse_prior_response_on_connection_failure(): call_count = 0 async def handler(request: httpx.Request) -> httpx.Response: nonlocal call_count call_count += 1 if call_count == 1: return httpx.Response(500, content=b'{"error":"first"}', request=request) raise httpx.ConnectError("connection failed", request=request) provider = OpenRouterTranscriptionProvider( settings=Settings(openrouter_api_key="test-key"), async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), ) with pytest.raises(ProviderError) as first_failure: await provider.transcribe(prompt_text="First", image_bytes=b"one", mime_type="image/png") with pytest.raises(ProviderError) as second_failure: await provider.transcribe(prompt_text="Second", image_bytes=b"two", mime_type="image/png") assert first_failure.value.transport_evidence is not None assert second_failure.value.transport_evidence is not None assert second_failure.value.transport_evidence.response_received is False assert second_failure.value.transport_evidence.body is None await provider.aclose() @pytest.mark.asyncio async def test_attempts_are_append_only_and_exported_with_integrity(default_session_factory): documents = DocumentService(session_factory=default_session_factory) jobs = JobService(session_factory=default_session_factory) sources = SourceService(session_factory=default_session_factory) document = await documents.create_document(Document(name="Evidence")) job = await jobs.create_job(Job(document_id=document.id)) source = await sources.create_source( Source( document_id=document.id, page_number=1, upload_name="page.png", filename="page.png", file_path="page.png", file_hash="a" * 64, file_size_bytes=10, ) ) await sources.create_job_source(JobSource(job_id=job.id, source_id=source.id)) now = datetime.now(UTC) await sources.update_job_source_transcription( job_id=job.id, source_id=source.id, text=None, error_detail="first failed", error_category="external_provider_error", failure_phase="connection", started_at=now, finished_at=now, ) await sources.update_job_source_transcription( job_id=job.id, source_id=source.id, text="second succeeded", provider="openrouter", model="vendor/model", started_at=now, finished_at=now, ) attempts = await sources.list_execution_attempts(source_id=source.id) assert [attempt.attempt_number for attempt in attempts] == [1, 2] assert attempts[0].status == JobSourceStatus.FAILED assert attempts[0].error_detail == "first failed" assert attempts[1].status == JobSourceStatus.TRANSCRIBED assert attempts[1].raw_transcription == "second succeeded" payload = {"words": [{"text": "second", "polygon": [0, 0, 1, 1]}]} payload_bytes = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() artifact = await sources.create_processing_artifact( ProcessingArtifact( source_id=source.id, execution_attempt_id=attempts[1].id, artifact_type="ocr.words", media_type="application/json", schema_name="example.ocr.words", schema_version="1", producer="fixture", producer_version="1", inline_payload=payload, payload_sha256=hashlib.sha256(payload_bytes).hexdigest(), byte_size=len(payload_bytes), coordinate_metadata={ "units": "normalized", "origin": "top-left", "width": 1, "height": 1, "transformations": [], }, ) ) export = await sources.build_evidence_export(source_id=source.id) assert export["source"]["digest_sha256"] == "a" * 64 assert [item["attempt_number"] for item in export["attempts"]] == [1, 2] assert export["artifacts"][0]["id"] == str(artifact.id) assert "file_path" not in json.dumps(export) with pytest.raises(JobDeleteBlockedError): await jobs.delete_job_with_guardrails(job_id=job.id) detail = await sources.read_source_detail(source.id) latest_job_source = detail.latest_job_source assert latest_job_source is not None assert latest_job_source.execution_attempts == [] assert detail.processing_artifacts == [] latest_attempt = await sources.read_latest_execution_attempt(job_source_id=latest_job_source.id) assert latest_attempt is not None assert latest_attempt.attempt_number == 2 def test_benchmark_scoring_preserves_literal_differences(): score = score_transcription( execution_attempt_id=uuid4(), reference="Farm house", candidate="farm house", assessment=EditorialAssessment(silent_normalizations=1), latency_ms=125, ) assert score.character_edits == 1 assert score.word_edits == 1 assert score.assessment.silent_normalizations == 1 @pytest.mark.asyncio async def test_large_json_artifact_uses_constrained_atomic_storage( default_session_factory, tmp_path, ): settings = Settings( openrouter_api_key="test-key", artifact_dir=tmp_path / "artifacts", artifact_inline_threshold_bytes=10, ) documents = DocumentService(session_factory=default_session_factory, settings=settings) sources = SourceService(session_factory=default_session_factory, settings=settings) document = await documents.create_document(Document(name="External Artifact")) source = await sources.create_source( Source( document_id=document.id, page_number=1, upload_name="page.png", filename="page.png", file_path="page.png", file_hash="b" * 64, file_size_bytes=10, ) ) artifact = await sources.create_json_artifact( source_id=source.id, execution_attempt_id=None, artifact_type="ocr.layout", schema_name="example.layout", schema_version="1", producer="fixture", producer_version="1", payload={"blocks": [{"text": "long enough to be external"}]}, ) assert artifact.inline_payload is None assert artifact.external_reference is not None stored_path = settings.artifact_dir / artifact.external_reference assert stored_path.is_file() assert hashlib.sha256(stored_path.read_bytes()).hexdigest() == artifact.payload_sha256 with pytest.raises(SourceDeleteBlockedError): await sources.delete_unlinked_source(source_id=source.id) stored_path.write_bytes(b'{"tampered":true}') with pytest.raises(TranscriptionError, match="integrity verification"): await sources.build_evidence_export(source_id=source.id) @pytest.mark.asyncio async def test_rejects_inline_artifact_with_incorrect_integrity(default_session_factory): documents = DocumentService(session_factory=default_session_factory) sources = SourceService(session_factory=default_session_factory) document = await documents.create_document(Document(name="Inline Integrity")) source = await sources.create_source( Source( document_id=document.id, page_number=1, upload_name="page.png", filename="page.png", file_path="page.png", file_hash="d" * 64, file_size_bytes=10, ) ) with pytest.raises(TranscriptionError, match="integrity verification"): await sources.create_processing_artifact( ProcessingArtifact( source_id=source.id, artifact_type="ocr.words", media_type="application/json", schema_name="example.words", schema_version="1", producer="fixture", producer_version="1", inline_payload={"words": []}, payload_sha256="0" * 64, byte_size=1, ) ) @pytest.mark.asyncio async def test_standalone_transcription_closes_locally_created_provider(tmp_path, monkeypatch): image_path = tmp_path / "page.png" image_path.write_bytes(b"image") class _Provider: closed = False async def transcribe(self, **kwargs): _ = kwargs return TranscriptionResult(text="result", provider="fixture", model="model") async def aclose(self): self.closed = True provider = _Provider() monkeypatch.setattr("transcription.services.sources.get_transcription_provider", lambda **_kwargs: provider) result = await transcribe_document_image( image_path, prompt_text="Prompt", settings=Settings(openrouter_api_key="test-key"), ) assert result.text == "result" assert provider.closed is True