transcription/tests/test_v42_evidence.py

"""Focused V4.2 evidence, integrity, and benchmark tests."""

from __future__ import annotations

import hashlib
import json
from datetime import UTC
from datetime import datetime
from uuid import uuid4

import httpx
import pytest
from pydantic import JsonValue

from transcription.benchmarking import EditorialAssessment
from transcription.benchmarking import score_transcription
from transcription.config import Settings
from transcription.db.models import Document
from transcription.db.models import Job
from transcription.db.models import JobSource
from transcription.db.models import JobSourceStatus
from transcription.db.models import Source
from transcription.providers.base import ProviderError
from transcription.providers.base import TranscriptionResult
from transcription.providers.evidence import SourceEvidenceReference
from transcription.providers.openrouter import OpenRouterTranscriptionProvider
from transcription.services.documents import DocumentService
from transcription.services.jobs import JobDeleteBlockedError
from transcription.services.jobs import JobService
from transcription.services.sources import SourceService
from transcription.services.sources import transcribe_document_image


def _json_object(value: JsonValue) -> dict[str, JsonValue]:
    """Narrow a JSON export member to an object, asserting the export shape."""
    assert isinstance(value, dict)
    return value


def _json_array(value: JsonValue) -> list[JsonValue]:
    """Narrow a JSON export member to an array, asserting the export shape."""
    assert isinstance(value, list)
    return value


class _ChunkedAsyncStream(httpx.AsyncByteStream):
    def __init__(self, chunks: list[bytes]):
        self._chunks = chunks

    async def __aiter__(self):
        for chunk in self._chunks:
            yield chunk

    async def aclose(self):
        return


@pytest.mark.asyncio
async def test_openrouter_captures_exact_transport_and_secret_safe_manifest():
    response_body = (
        b'{"id":"gen-1","created":1,"model":"vendor/model","object":"chat.completion",'
        b'"system_fingerprint":null,"choices":[{"index":0,"finish_reason":"stop",'
        b'"message":{"role":"assistant","content":"Transcript"}}],'
        b'"unknown_transport_field":{"retained":true}}'
    )

    async def handler(request: httpx.Request) -> httpx.Response:
        assert request.headers["authorization"] == "Bearer test-key"
        return httpx.Response(
            200,
            content=response_body,
            headers={
                "Content-Type": "application/json",
                "X-Request-Id": "req-123",
                "Set-Cookie": "must-not-persist=1",
            },
            request=request,
        )

    async_client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
    provider = OpenRouterTranscriptionProvider(
        settings=Settings(openrouter_api_key="test-key"),
        async_client=async_client,
    )
    result = await provider.transcribe(
        prompt_text="Literal prompt",
        image_bytes=b"source-bytes",
        mime_type="image/png",
        source_reference=SourceEvidenceReference(
            source_id=uuid4(),
            digest_sha256=hashlib.sha256(b"source-bytes").hexdigest(),
            byte_size=len(b"source-bytes"),
            media_type="image/png",
            page_number=1,
        ),
    )

    assert result.transport_evidence is not None
    assert result.transport_evidence.body == response_body
    assert result.transport_evidence.safe_headers == {
        "content-type": "application/json",
        "x-request-id": "req-123",
    }
    assert b'"unknown_transport_field":{"retained":true}' in result.transport_evidence.body
    assert result.request_manifest is not None
    serialized_manifest = json.dumps(result.request_manifest.model_dump(mode="json"))
    assert "data:image/png;base64" not in serialized_manifest
    assert "test-key" not in serialized_manifest
    assert result.request_manifest.omitted_optional_parameters == ("temperature", "top_p")


@pytest.mark.asyncio
async def test_openrouter_captures_body_consumed_as_sdk_stream():
    response_body = (
        b'{"id":"gen-2","created":1,"model":"vendor/model","object":"chat.completion",'
        b'"system_fingerprint":null,"choices":[{"index":0,"finish_reason":"stop",'
        b'"message":{"role":"assistant","content":"Transcript"}}],'
        b'"unknown_streamed_field":true}'
    )

    async def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(
            200,
            stream=_ChunkedAsyncStream([response_body[:23], response_body[23:61], response_body[61:]]),
            headers={"Content-Type": "application/json"},
            request=request,
        )

    provider = OpenRouterTranscriptionProvider(
        settings=Settings(openrouter_api_key="test-key"),
        async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
    )
    result = await provider.transcribe(
        prompt_text="Literal prompt",
        image_bytes=b"source-bytes",
        mime_type="image/png",
    )

    assert result.transport_evidence is not None
    assert result.transport_evidence.body == response_body
    assert b'"unknown_streamed_field":true' in result.transport_evidence.body
    await provider.aclose()


@pytest.mark.asyncio
async def test_openrouter_failure_retains_safe_response_evidence():
    async def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(
            500,
            content=b'{"error":{"message":"provider unavailable"}}',
            headers={"Content-Type": "application/json", "Retry-After": "2", "Set-Cookie": "secret=1"},
            request=request,
        )

    provider = OpenRouterTranscriptionProvider(
        settings=Settings(openrouter_api_key="test-key"),
        async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
    )
    with pytest.raises(ProviderError) as failure:
        await provider.transcribe(
            prompt_text="Literal prompt",
            image_bytes=b"source-bytes",
            mime_type="image/png",
        )

    evidence = failure.value.transport_evidence
    assert evidence is not None
    assert evidence.status_code == 500
    assert evidence.body == b'{"error":{"message":"provider unavailable"}}'
    assert evidence.safe_headers == {"content-type": "application/json", "retry-after": "2"}
    assert str(failure.value) == "OpenRouter request failed with HTTP 500: provider unavailable"


@pytest.mark.asyncio
async def test_openrouter_does_not_reuse_prior_response_on_connection_failure():
    call_count = 0

    async def handler(request: httpx.Request) -> httpx.Response:
        nonlocal call_count
        call_count += 1
        if call_count == 1:
            return httpx.Response(500, content=b'{"error":"first"}', request=request)
        raise httpx.ConnectError("connection failed", request=request)

    provider = OpenRouterTranscriptionProvider(
        settings=Settings(openrouter_api_key="test-key"),
        async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
    )
    with pytest.raises(ProviderError) as first_failure:
        await provider.transcribe(prompt_text="First", image_bytes=b"one", mime_type="image/png")
    with pytest.raises(ProviderError) as second_failure:
        await provider.transcribe(prompt_text="Second", image_bytes=b"two", mime_type="image/png")

    assert first_failure.value.transport_evidence is not None
    assert second_failure.value.transport_evidence is not None
    assert second_failure.value.transport_evidence.response_received is False
    assert second_failure.value.transport_evidence.body is None
    await provider.aclose()


@pytest.mark.asyncio
async def test_attempts_are_append_only_and_exported_with_integrity(default_session_factory):
    documents = DocumentService(session_factory=default_session_factory)
    jobs = JobService(session_factory=default_session_factory)
    sources = SourceService(session_factory=default_session_factory)
    document = await documents.create_document(Document(name="Evidence"))
    job = await jobs.create_job(Job(document_id=document.id))
    source = await sources.create_source(
        Source(
            document_id=document.id,
            page_number=1,
            upload_name="page.png",
            filename="page.png",
            file_path="page.png",
            file_hash="a" * 64,
            file_size_bytes=10,
        )
    )
    await sources.create_job_source(JobSource(job_id=job.id, source_id=source.id))
    now = datetime.now(UTC)

    await sources.update_job_source_transcription(
        job_id=job.id,
        source_id=source.id,
        text=None,
        error_detail="first failed",
        error_category="external_provider_error",
        failure_phase="connection",
        started_at=now,
        finished_at=now,
    )
    await sources.update_job_source_transcription(
        job_id=job.id,
        source_id=source.id,
        text="second succeeded",
        provider="openrouter",
        model="vendor/model",
        started_at=now,
        finished_at=now,
    )

    attempts = await sources.list_execution_attempts(source_id=source.id)
    assert [attempt.attempt_number for attempt in attempts] == [1, 2]
    assert attempts[0].status == JobSourceStatus.FAILED
    assert attempts[0].error_detail == "first failed"
    assert attempts[1].status == JobSourceStatus.TRANSCRIBED
    assert attempts[1].raw_transcription == "second succeeded"

    export = await sources.build_evidence_export(source_id=source.id)
    assert _json_object(export["source"])["digest_sha256"] == "a" * 64
    assert [_json_object(item)["attempt_number"] for item in _json_array(export["attempts"])] == [1, 2]
    assert "file_path" not in json.dumps(export)
    with pytest.raises(JobDeleteBlockedError):
        await jobs.delete_job_with_guardrails(job_id=job.id)

    detail = await sources.read_source_detail(source.id)
    latest_job_source = detail.latest_job_source
    assert latest_job_source is not None
    assert latest_job_source.execution_attempts == []
    latest_attempt = await sources.read_latest_execution_attempt(job_source_id=latest_job_source.id)
    assert latest_attempt is not None
    assert latest_attempt.attempt.attempt_number == 2


def test_benchmark_scoring_preserves_literal_differences():
    score = score_transcription(
        execution_attempt_id=uuid4(),
        reference="Farm house",
        candidate="farm house",
        assessment=EditorialAssessment(silent_normalizations=1),
        latency_ms=125,
    )
    assert score.character_edits == 1
    assert score.word_edits == 1
    assert score.assessment.silent_normalizations == 1


@pytest.mark.asyncio
async def test_standalone_transcription_closes_locally_created_provider(tmp_path, monkeypatch):
    image_path = tmp_path / "page.png"
    image_path.write_bytes(b"image")

    class _Provider:
        closed = False

        async def transcribe(self, **kwargs):
            _ = kwargs
            return TranscriptionResult(text="result", provider="fixture", model="model")

        async def aclose(self):
            self.closed = True

    provider = _Provider()
    monkeypatch.setattr("transcription.services.sources.get_transcription_provider", lambda **_kwargs: provider)

    result = await transcribe_document_image(
        image_path,
        prompt_text="Prompt",
        settings=Settings(openrouter_api_key="test-key"),
    )

    assert result.text == "result"
    assert provider.closed is True