generated from john/python-template
zoltan57andCopilot App
66e2dce465
Baseline was 207 diagnostics. Two real bugs were hiding in the noise: - tools/run_destructive_tests.py imported ctypes.wintypes at module scope, which raises on non-Windows, and called fcntl unconditionally. The Windows and POSIX implementations now live under a module-level sys.platform split. - tests/ui/test_sources_page.py constructed Source(...) without document_id. Structural fixes, not suppressions: - New src/transcription/db/loading.py owns the SQLModel-field to QueryableAttribute reinterpretation via orm_attribute()/selectinload()/ defer(). This removed 42 "# pyright: ignore[reportArgumentType]" comments across documents/jobs/people/sources. Its docstring records that selectinload(A.b, B.c) is NOT equivalent to the chained form: varargs applies the selectin strategy only to the last path element, which under lazy="raise" raises InvalidRequestError at render time. - db/session.py transaction_scope no longer accepts or yields AsyncSessionTransaction. No caller ever passed one, sessionmaker.begin() yields an AsyncSession, and the dead branch was latently buggy because services call .exec(). Cleared 7 workflows.py diagnostics. - services/registry.py RegistryService is bound by a new RegistryEntry Protocol instead of bare SQLModel, so the shared implementation can read id/label/normalized_label/is_active. Cleared 9 diagnostics. - Column expressions in sources.py/jobs.py/test_store.py wrap in sqlmodel col(), the idiom already used in registry.py. - read_source_navigation wraps its literal tuple bounds in literal(). - normalization.py narrows with isinstance(image, TiffImageFile) rather than comparing image.format, since tag_v2 is TIFF-only. - linked_people.render uses @ui.refreshable_method, the NiceGUI API for bound methods. - The OpenRouter capturing client re-raises ResponseNotRead when the response stream is not async rather than mis-wrapping it. Tooling gate: - New .pre-commit-config.yaml runs ruff check and ty check as blocking hooks. No pre-commit config previously existed. Negative-tested: injecting a type error fails both hooks. - The last two "# pyright: ignore" comments (config.py) are removed; ty does not honor pyright directives. One "# ty: ignore" remains, in tests/test_prompts.py, where the test deliberately assigns to a frozen field to assert ValidationError. - asyncio_default_fixture_loop_scope is pinned to "function" so pytest-asyncio behavior does not shift on upgrade. Verification: ruff check clean, ty check reports 0 diagnostics, 292 passed and 4 skipped, pre-commit passes and demonstrably fails on a regression, and tools/run_destructive_tests.py runs on Windows. Co-authored-by: Copilot App <[email protected]>
416 lines
16 KiB
Python
416 lines
16 KiB
Python
"""Focused V4.2 evidence, integrity, and benchmark tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from datetime import UTC
|
|
from datetime import datetime
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
import pytest
|
|
from pydantic import JsonValue
|
|
|
|
from transcription.benchmarking import EditorialAssessment
|
|
from transcription.benchmarking import score_transcription
|
|
from transcription.config import Settings
|
|
from transcription.db.models import Document
|
|
from transcription.db.models import Job
|
|
from transcription.db.models import JobSource
|
|
from transcription.db.models import JobSourceStatus
|
|
from transcription.db.models import ProcessingArtifact
|
|
from transcription.db.models import Source
|
|
from transcription.providers.base import ProviderError
|
|
from transcription.providers.base import TranscriptionResult
|
|
from transcription.providers.evidence import SourceEvidenceReference
|
|
from transcription.providers.openrouter import OpenRouterTranscriptionProvider
|
|
from transcription.services.documents import DocumentService
|
|
from transcription.services.jobs import JobDeleteBlockedError
|
|
from transcription.services.jobs import JobService
|
|
from transcription.services.sources import SourceDeleteBlockedError
|
|
from transcription.services.sources import SourceService
|
|
from transcription.services.sources import TranscriptionError
|
|
from transcription.services.sources import transcribe_document_image
|
|
|
|
|
|
def _json_object(value: JsonValue) -> dict[str, JsonValue]:
|
|
"""Narrow a JSON export member to an object, asserting the export shape."""
|
|
assert isinstance(value, dict)
|
|
return value
|
|
|
|
|
|
def _json_array(value: JsonValue) -> list[JsonValue]:
|
|
"""Narrow a JSON export member to an array, asserting the export shape."""
|
|
assert isinstance(value, list)
|
|
return value
|
|
|
|
|
|
class _ChunkedAsyncStream(httpx.AsyncByteStream):
|
|
def __init__(self, chunks: list[bytes]):
|
|
self._chunks = chunks
|
|
|
|
async def __aiter__(self):
|
|
for chunk in self._chunks:
|
|
yield chunk
|
|
|
|
async def aclose(self):
|
|
return
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_openrouter_captures_exact_transport_and_secret_safe_manifest():
|
|
response_body = (
|
|
b'{"id":"gen-1","created":1,"model":"vendor/model","object":"chat.completion",'
|
|
b'"system_fingerprint":null,"choices":[{"index":0,"finish_reason":"stop",'
|
|
b'"message":{"role":"assistant","content":"Transcript"}}],'
|
|
b'"unknown_transport_field":{"retained":true}}'
|
|
)
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
assert request.headers["authorization"] == "Bearer test-key"
|
|
return httpx.Response(
|
|
200,
|
|
content=response_body,
|
|
headers={
|
|
"Content-Type": "application/json",
|
|
"X-Request-Id": "req-123",
|
|
"Set-Cookie": "must-not-persist=1",
|
|
},
|
|
request=request,
|
|
)
|
|
|
|
async_client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
provider = OpenRouterTranscriptionProvider(
|
|
settings=Settings(openrouter_api_key="test-key"),
|
|
async_client=async_client,
|
|
)
|
|
result = await provider.transcribe(
|
|
prompt_text="Literal prompt",
|
|
image_bytes=b"source-bytes",
|
|
mime_type="image/png",
|
|
source_reference=SourceEvidenceReference(
|
|
source_id=uuid4(),
|
|
digest_sha256=hashlib.sha256(b"source-bytes").hexdigest(),
|
|
byte_size=len(b"source-bytes"),
|
|
media_type="image/png",
|
|
page_number=1,
|
|
),
|
|
)
|
|
|
|
assert result.transport_evidence is not None
|
|
assert result.transport_evidence.body == response_body
|
|
assert result.transport_evidence.safe_headers == {
|
|
"content-type": "application/json",
|
|
"x-request-id": "req-123",
|
|
}
|
|
assert b'"unknown_transport_field":{"retained":true}' in result.transport_evidence.body
|
|
assert result.request_manifest is not None
|
|
serialized_manifest = json.dumps(result.request_manifest.model_dump(mode="json"))
|
|
assert "data:image/png;base64" not in serialized_manifest
|
|
assert "test-key" not in serialized_manifest
|
|
assert result.request_manifest.omitted_optional_parameters == ("temperature", "top_p")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_openrouter_captures_body_consumed_as_sdk_stream():
|
|
response_body = (
|
|
b'{"id":"gen-2","created":1,"model":"vendor/model","object":"chat.completion",'
|
|
b'"system_fingerprint":null,"choices":[{"index":0,"finish_reason":"stop",'
|
|
b'"message":{"role":"assistant","content":"Transcript"}}],'
|
|
b'"unknown_streamed_field":true}'
|
|
)
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
200,
|
|
stream=_ChunkedAsyncStream([response_body[:23], response_body[23:61], response_body[61:]]),
|
|
headers={"Content-Type": "application/json"},
|
|
request=request,
|
|
)
|
|
|
|
provider = OpenRouterTranscriptionProvider(
|
|
settings=Settings(openrouter_api_key="test-key"),
|
|
async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
|
)
|
|
result = await provider.transcribe(
|
|
prompt_text="Literal prompt",
|
|
image_bytes=b"source-bytes",
|
|
mime_type="image/png",
|
|
)
|
|
|
|
assert result.transport_evidence is not None
|
|
assert result.transport_evidence.body == response_body
|
|
assert b'"unknown_streamed_field":true' in result.transport_evidence.body
|
|
await provider.aclose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_openrouter_failure_retains_safe_response_evidence():
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
500,
|
|
content=b'{"error":{"message":"provider unavailable"}}',
|
|
headers={"Content-Type": "application/json", "Retry-After": "2", "Set-Cookie": "secret=1"},
|
|
request=request,
|
|
)
|
|
|
|
provider = OpenRouterTranscriptionProvider(
|
|
settings=Settings(openrouter_api_key="test-key"),
|
|
async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
|
)
|
|
with pytest.raises(ProviderError) as failure:
|
|
await provider.transcribe(
|
|
prompt_text="Literal prompt",
|
|
image_bytes=b"source-bytes",
|
|
mime_type="image/png",
|
|
)
|
|
|
|
evidence = failure.value.transport_evidence
|
|
assert evidence is not None
|
|
assert evidence.status_code == 500
|
|
assert evidence.body == b'{"error":{"message":"provider unavailable"}}'
|
|
assert evidence.safe_headers == {"content-type": "application/json", "retry-after": "2"}
|
|
assert str(failure.value) == "OpenRouter request failed with HTTP 500: provider unavailable"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_openrouter_does_not_reuse_prior_response_on_connection_failure():
|
|
call_count = 0
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
nonlocal call_count
|
|
call_count += 1
|
|
if call_count == 1:
|
|
return httpx.Response(500, content=b'{"error":"first"}', request=request)
|
|
raise httpx.ConnectError("connection failed", request=request)
|
|
|
|
provider = OpenRouterTranscriptionProvider(
|
|
settings=Settings(openrouter_api_key="test-key"),
|
|
async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
|
)
|
|
with pytest.raises(ProviderError) as first_failure:
|
|
await provider.transcribe(prompt_text="First", image_bytes=b"one", mime_type="image/png")
|
|
with pytest.raises(ProviderError) as second_failure:
|
|
await provider.transcribe(prompt_text="Second", image_bytes=b"two", mime_type="image/png")
|
|
|
|
assert first_failure.value.transport_evidence is not None
|
|
assert second_failure.value.transport_evidence is not None
|
|
assert second_failure.value.transport_evidence.response_received is False
|
|
assert second_failure.value.transport_evidence.body is None
|
|
await provider.aclose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_attempts_are_append_only_and_exported_with_integrity(default_session_factory):
|
|
documents = DocumentService(session_factory=default_session_factory)
|
|
jobs = JobService(session_factory=default_session_factory)
|
|
sources = SourceService(session_factory=default_session_factory)
|
|
document = await documents.create_document(Document(name="Evidence"))
|
|
job = await jobs.create_job(Job(document_id=document.id))
|
|
source = await sources.create_source(
|
|
Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="page.png",
|
|
filename="page.png",
|
|
file_path="page.png",
|
|
file_hash="a" * 64,
|
|
file_size_bytes=10,
|
|
)
|
|
)
|
|
await sources.create_job_source(JobSource(job_id=job.id, source_id=source.id))
|
|
now = datetime.now(UTC)
|
|
|
|
await sources.update_job_source_transcription(
|
|
job_id=job.id,
|
|
source_id=source.id,
|
|
text=None,
|
|
error_detail="first failed",
|
|
error_category="external_provider_error",
|
|
failure_phase="connection",
|
|
started_at=now,
|
|
finished_at=now,
|
|
)
|
|
await sources.update_job_source_transcription(
|
|
job_id=job.id,
|
|
source_id=source.id,
|
|
text="second succeeded",
|
|
provider="openrouter",
|
|
model="vendor/model",
|
|
started_at=now,
|
|
finished_at=now,
|
|
)
|
|
|
|
attempts = await sources.list_execution_attempts(source_id=source.id)
|
|
assert [attempt.attempt_number for attempt in attempts] == [1, 2]
|
|
assert attempts[0].status == JobSourceStatus.FAILED
|
|
assert attempts[0].error_detail == "first failed"
|
|
assert attempts[1].status == JobSourceStatus.TRANSCRIBED
|
|
assert attempts[1].raw_transcription == "second succeeded"
|
|
|
|
payload = {"words": [{"text": "second", "polygon": [0, 0, 1, 1]}]}
|
|
payload_bytes = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
|
|
artifact = await sources.create_processing_artifact(
|
|
ProcessingArtifact(
|
|
source_id=source.id,
|
|
execution_attempt_id=attempts[1].id,
|
|
artifact_type="ocr.words",
|
|
media_type="application/json",
|
|
schema_name="example.ocr.words",
|
|
schema_version="1",
|
|
producer="fixture",
|
|
producer_version="1",
|
|
inline_payload=payload,
|
|
payload_sha256=hashlib.sha256(payload_bytes).hexdigest(),
|
|
byte_size=len(payload_bytes),
|
|
coordinate_metadata={
|
|
"units": "normalized",
|
|
"origin": "top-left",
|
|
"width": 1,
|
|
"height": 1,
|
|
"transformations": [],
|
|
},
|
|
)
|
|
)
|
|
export = await sources.build_evidence_export(source_id=source.id)
|
|
assert _json_object(export["source"])["digest_sha256"] == "a" * 64
|
|
assert [_json_object(item)["attempt_number"] for item in _json_array(export["attempts"])] == [1, 2]
|
|
assert _json_object(_json_array(export["artifacts"])[0])["id"] == str(artifact.id)
|
|
assert "file_path" not in json.dumps(export)
|
|
with pytest.raises(JobDeleteBlockedError):
|
|
await jobs.delete_job_with_guardrails(job_id=job.id)
|
|
|
|
detail = await sources.read_source_detail(source.id)
|
|
latest_job_source = detail.latest_job_source
|
|
assert latest_job_source is not None
|
|
assert latest_job_source.execution_attempts == []
|
|
assert detail.processing_artifacts == []
|
|
latest_attempt = await sources.read_latest_execution_attempt(job_source_id=latest_job_source.id)
|
|
assert latest_attempt is not None
|
|
assert latest_attempt.attempt.attempt_number == 2
|
|
|
|
|
|
def test_benchmark_scoring_preserves_literal_differences():
|
|
score = score_transcription(
|
|
execution_attempt_id=uuid4(),
|
|
reference="Farm house",
|
|
candidate="farm house",
|
|
assessment=EditorialAssessment(silent_normalizations=1),
|
|
latency_ms=125,
|
|
)
|
|
assert score.character_edits == 1
|
|
assert score.word_edits == 1
|
|
assert score.assessment.silent_normalizations == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_large_json_artifact_uses_constrained_atomic_storage(
|
|
default_session_factory,
|
|
tmp_path,
|
|
):
|
|
settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
artifact_dir=tmp_path / "artifacts",
|
|
artifact_inline_threshold_bytes=10,
|
|
)
|
|
documents = DocumentService(session_factory=default_session_factory, settings=settings)
|
|
sources = SourceService(session_factory=default_session_factory, settings=settings)
|
|
document = await documents.create_document(Document(name="External Artifact"))
|
|
source = await sources.create_source(
|
|
Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="page.png",
|
|
filename="page.png",
|
|
file_path="page.png",
|
|
file_hash="b" * 64,
|
|
file_size_bytes=10,
|
|
)
|
|
)
|
|
|
|
artifact = await sources.create_json_artifact(
|
|
source_id=source.id,
|
|
execution_attempt_id=None,
|
|
artifact_type="ocr.layout",
|
|
schema_name="example.layout",
|
|
schema_version="1",
|
|
producer="fixture",
|
|
producer_version="1",
|
|
payload={"blocks": [{"text": "long enough to be external"}]},
|
|
)
|
|
|
|
assert artifact.inline_payload is None
|
|
assert artifact.external_reference is not None
|
|
stored_path = settings.artifact_dir / artifact.external_reference
|
|
assert stored_path.is_file()
|
|
assert hashlib.sha256(stored_path.read_bytes()).hexdigest() == artifact.payload_sha256
|
|
with pytest.raises(SourceDeleteBlockedError):
|
|
await sources.delete_unlinked_source(source_id=source.id)
|
|
|
|
stored_path.write_bytes(b'{"tampered":true}')
|
|
with pytest.raises(TranscriptionError, match="integrity verification"):
|
|
await sources.build_evidence_export(source_id=source.id)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rejects_inline_artifact_with_incorrect_integrity(default_session_factory):
|
|
documents = DocumentService(session_factory=default_session_factory)
|
|
sources = SourceService(session_factory=default_session_factory)
|
|
document = await documents.create_document(Document(name="Inline Integrity"))
|
|
source = await sources.create_source(
|
|
Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="page.png",
|
|
filename="page.png",
|
|
file_path="page.png",
|
|
file_hash="d" * 64,
|
|
file_size_bytes=10,
|
|
)
|
|
)
|
|
|
|
with pytest.raises(TranscriptionError, match="integrity verification"):
|
|
await sources.create_processing_artifact(
|
|
ProcessingArtifact(
|
|
source_id=source.id,
|
|
artifact_type="ocr.words",
|
|
media_type="application/json",
|
|
schema_name="example.words",
|
|
schema_version="1",
|
|
producer="fixture",
|
|
producer_version="1",
|
|
inline_payload={"words": []},
|
|
payload_sha256="0" * 64,
|
|
byte_size=1,
|
|
)
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_standalone_transcription_closes_locally_created_provider(tmp_path, monkeypatch):
|
|
image_path = tmp_path / "page.png"
|
|
image_path.write_bytes(b"image")
|
|
|
|
class _Provider:
|
|
closed = False
|
|
|
|
async def transcribe(self, **kwargs):
|
|
_ = kwargs
|
|
return TranscriptionResult(text="result", provider="fixture", model="model")
|
|
|
|
async def aclose(self):
|
|
self.closed = True
|
|
|
|
provider = _Provider()
|
|
monkeypatch.setattr("transcription.services.sources.get_transcription_provider", lambda **_kwargs: provider)
|
|
|
|
result = await transcribe_document_image(
|
|
image_path,
|
|
prompt_text="Prompt",
|
|
settings=Settings(openrouter_api_key="test-key"),
|
|
)
|
|
|
|
assert result.text == "result"
|
|
assert provider.closed is True
|