"""Focused V4.2 evidence, integrity, and benchmark tests.""" from __future__ import annotations import hashlib import json from datetime import UTC from datetime import datetime from uuid import uuid4 import httpx import pytest from pydantic import JsonValue from transcription.benchmarking import EditorialAssessment from transcription.benchmarking import score_transcription from transcription.config import Settings from transcription.db.models import Document from transcription.db.models import Job from transcription.db.models import JobSource from transcription.db.models import JobSourceStatus from transcription.db.models import Source from transcription.providers.base import ProviderError from transcription.providers.base import TranscriptionResult from transcription.providers.evidence import SourceEvidenceReference from transcription.providers.openrouter import OpenRouterTranscriptionProvider from transcription.services.documents import DocumentService from transcription.services.jobs import JobDeleteBlockedError from transcription.services.jobs import JobService from transcription.services.sources import SourceService from transcription.services.sources import transcribe_document_image def _json_object(value: JsonValue) -> dict[str, JsonValue]: """Narrow a JSON export member to an object, asserting the export shape.""" assert isinstance(value, dict) return value def _json_array(value: JsonValue) -> list[JsonValue]: """Narrow a JSON export member to an array, asserting the export shape.""" assert isinstance(value, list) return value class _ChunkedAsyncStream(httpx.AsyncByteStream): def __init__(self, chunks: list[bytes]): self._chunks = chunks async def __aiter__(self): for chunk in self._chunks: yield chunk async def aclose(self): return @pytest.mark.asyncio async def test_openrouter_captures_exact_transport_and_secret_safe_manifest(): response_body = ( b'{"id":"gen-1","created":1,"model":"vendor/model","object":"chat.completion",' b'"system_fingerprint":null,"choices":[{"index":0,"finish_reason":"stop",' b'"message":{"role":"assistant","content":"Transcript"}}],' b'"unknown_transport_field":{"retained":true}}' ) async def handler(request: httpx.Request) -> httpx.Response: assert request.headers["authorization"] == "Bearer test-key" return httpx.Response( 200, content=response_body, headers={ "Content-Type": "application/json", "X-Request-Id": "req-123", "Set-Cookie": "must-not-persist=1", }, request=request, ) async_client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) provider = OpenRouterTranscriptionProvider( settings=Settings(openrouter_api_key="test-key"), async_client=async_client, ) result = await provider.transcribe( prompt_text="Literal prompt", image_bytes=b"source-bytes", mime_type="image/png", source_reference=SourceEvidenceReference( source_id=uuid4(), digest_sha256=hashlib.sha256(b"source-bytes").hexdigest(), byte_size=len(b"source-bytes"), media_type="image/png", page_number=1, ), ) assert result.transport_evidence is not None assert result.transport_evidence.body == response_body assert result.transport_evidence.safe_headers == { "content-type": "application/json", "x-request-id": "req-123", } assert b'"unknown_transport_field":{"retained":true}' in result.transport_evidence.body assert result.request_manifest is not None serialized_manifest = json.dumps(result.request_manifest.model_dump(mode="json")) assert "data:image/png;base64" not in serialized_manifest assert "test-key" not in serialized_manifest assert result.request_manifest.omitted_optional_parameters == ("temperature", "top_p") @pytest.mark.asyncio async def test_openrouter_captures_body_consumed_as_sdk_stream(): response_body = ( b'{"id":"gen-2","created":1,"model":"vendor/model","object":"chat.completion",' b'"system_fingerprint":null,"choices":[{"index":0,"finish_reason":"stop",' b'"message":{"role":"assistant","content":"Transcript"}}],' b'"unknown_streamed_field":true}' ) async def handler(request: httpx.Request) -> httpx.Response: return httpx.Response( 200, stream=_ChunkedAsyncStream([response_body[:23], response_body[23:61], response_body[61:]]), headers={"Content-Type": "application/json"}, request=request, ) provider = OpenRouterTranscriptionProvider( settings=Settings(openrouter_api_key="test-key"), async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), ) result = await provider.transcribe( prompt_text="Literal prompt", image_bytes=b"source-bytes", mime_type="image/png", ) assert result.transport_evidence is not None assert result.transport_evidence.body == response_body assert b'"unknown_streamed_field":true' in result.transport_evidence.body await provider.aclose() @pytest.mark.asyncio async def test_openrouter_failure_retains_safe_response_evidence(): async def handler(request: httpx.Request) -> httpx.Response: return httpx.Response( 500, content=b'{"error":{"message":"provider unavailable"}}', headers={"Content-Type": "application/json", "Retry-After": "2", "Set-Cookie": "secret=1"}, request=request, ) provider = OpenRouterTranscriptionProvider( settings=Settings(openrouter_api_key="test-key"), async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), ) with pytest.raises(ProviderError) as failure: await provider.transcribe( prompt_text="Literal prompt", image_bytes=b"source-bytes", mime_type="image/png", ) evidence = failure.value.transport_evidence assert evidence is not None assert evidence.status_code == 500 assert evidence.body == b'{"error":{"message":"provider unavailable"}}' assert evidence.safe_headers == {"content-type": "application/json", "retry-after": "2"} assert str(failure.value) == "OpenRouter request failed with HTTP 500: provider unavailable" @pytest.mark.asyncio async def test_openrouter_does_not_reuse_prior_response_on_connection_failure(): call_count = 0 async def handler(request: httpx.Request) -> httpx.Response: nonlocal call_count call_count += 1 if call_count == 1: return httpx.Response(500, content=b'{"error":"first"}', request=request) raise httpx.ConnectError("connection failed", request=request) provider = OpenRouterTranscriptionProvider( settings=Settings(openrouter_api_key="test-key"), async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), ) with pytest.raises(ProviderError) as first_failure: await provider.transcribe(prompt_text="First", image_bytes=b"one", mime_type="image/png") with pytest.raises(ProviderError) as second_failure: await provider.transcribe(prompt_text="Second", image_bytes=b"two", mime_type="image/png") assert first_failure.value.transport_evidence is not None assert second_failure.value.transport_evidence is not None assert second_failure.value.transport_evidence.response_received is False assert second_failure.value.transport_evidence.body is None await provider.aclose() @pytest.mark.asyncio async def test_attempts_are_append_only_and_exported_with_integrity(default_session_factory): documents = DocumentService(session_factory=default_session_factory) jobs = JobService(session_factory=default_session_factory) sources = SourceService(session_factory=default_session_factory) document = await documents.create_document(Document(name="Evidence")) job = await jobs.create_job(Job(document_id=document.id)) source = await sources.create_source( Source( document_id=document.id, page_number=1, upload_name="page.png", filename="page.png", file_path="page.png", file_hash="a" * 64, file_size_bytes=10, ) ) await sources.create_job_source(JobSource(job_id=job.id, source_id=source.id)) now = datetime.now(UTC) await sources.update_job_source_transcription( job_id=job.id, source_id=source.id, text=None, error_detail="first failed", error_category="external_provider_error", failure_phase="connection", started_at=now, finished_at=now, ) await sources.update_job_source_transcription( job_id=job.id, source_id=source.id, text="second succeeded", provider="openrouter", model="vendor/model", started_at=now, finished_at=now, ) attempts = await sources.list_execution_attempts(source_id=source.id) assert [attempt.attempt_number for attempt in attempts] == [1, 2] assert attempts[0].status == JobSourceStatus.FAILED assert attempts[0].error_detail == "first failed" assert attempts[1].status == JobSourceStatus.TRANSCRIBED assert attempts[1].raw_transcription == "second succeeded" export = await sources.build_evidence_export(source_id=source.id) assert _json_object(export["source"])["digest_sha256"] == "a" * 64 assert [_json_object(item)["attempt_number"] for item in _json_array(export["attempts"])] == [1, 2] assert "file_path" not in json.dumps(export) with pytest.raises(JobDeleteBlockedError): await jobs.delete_job_with_guardrails(job_id=job.id) detail = await sources.read_source_detail(source.id) latest_job_source = detail.latest_job_source assert latest_job_source is not None assert latest_job_source.execution_attempts == [] latest_attempt = await sources.read_latest_execution_attempt(job_source_id=latest_job_source.id) assert latest_attempt is not None assert latest_attempt.attempt.attempt_number == 2 def test_benchmark_scoring_preserves_literal_differences(): score = score_transcription( execution_attempt_id=uuid4(), reference="Farm house", candidate="farm house", assessment=EditorialAssessment(silent_normalizations=1), latency_ms=125, ) assert score.character_edits == 1 assert score.word_edits == 1 assert score.assessment.silent_normalizations == 1 @pytest.mark.asyncio async def test_standalone_transcription_closes_locally_created_provider(tmp_path, monkeypatch): image_path = tmp_path / "page.png" image_path.write_bytes(b"image") class _Provider: closed = False async def transcribe(self, **kwargs): _ = kwargs return TranscriptionResult(text="result", provider="fixture", model="model") async def aclose(self): self.closed = True provider = _Provider() monkeypatch.setattr("transcription.services.sources.get_transcription_provider", lambda **_kwargs: provider) result = await transcribe_document_image( image_path, prompt_text="Prompt", settings=Settings(openrouter_api_key="test-key"), ) assert result.text == "result" assert provider.closed is True