generated from john/python-template
zoltan57andCopilot App
7dd0d2c9bf
Decompose SourceService along the aggregate boundary and then correct the
instruction file that caused it to grow, in that order. The refactor is the
empirical test of the rule.
services/evidence.py (new)
EvidenceService owns ExecutionAttempt: read_latest_execution_attempt,
list_execution_attempts, promote_machine_attempt, build_evidence_export,
plus the LatestExecutionAttempt projection. Moved verbatim from sources.py.
services/errors.py (new)
The five-class error hierarchy (PromptLoadError, TranscriptionError,
TranscriptionNotFoundError, SourceDeleteBlockedError,
CandidatePromotionError) moved out of sources.py. evidence.py needs
TranscriptionNotFoundError, and test_service_boundaries.py correctly
rejected the sibling import. errors.py defines no *Service class, so it is
a legal shared home. This was the boundary test doing its job, not an
obstacle to route around.
sources.py 1,389 -> 885 lines (1,063 after Phase 2).
services/__init__.py
ServiceBundle and from_session_factory register evidence. Note that
field-by-field ServiceBundle construction silently binds services to the
process-global session factory via default_factory; from_session_factory is
the only safe constructor. Two test bundles were fixed for this.
.github/instructions/services.instructions.md
Rewritten to describe the boundaries the decomposition actually produced,
per plan Phase 3 task 7 and review log [59].
- "1 service class per data model" -> one service class per aggregate.
The table-shaped rule is the measured cause of sources.py reaching
1,389 lines; DocumentType has no lifecycle without Document.
- New Model Ownership section. Junctions are owned by their lifecycle
owner, the service that creates and deletes the rows: document_person
to PeopleService (sole writer, measured), job_source to SourceService.
Two carve-outs are stated rather than left as silent violations:
cascade deletion when a service deletes its own aggregate root, and
status transitions that create and delete nothing (cancel_job,
resubmit_failed_sources), which are Job lifecycle events on the work
queue. EvidenceService.promote_machine_attempt's two-field write to
Source is named and scoped.
- Mandatory CRUD softened to intent. It was already false: five modules
define no service class, EvidenceService has no create/delete because
ExecutionAttempt is append-only, RegistryService uses <op>_entry.
- Separated reading across models via eager loads from the owning root,
which is allowed, from importing another service, which is not. The old
line 13 and lines 75-77 read as contradictory.
- Typo: picutre.
No code was moved to satisfy the rule.
tests/test_service_boundaries.py
Docstring no longer cites the instruction file by line number; that anchor
would desynchronise silently. errors.py added to the neutral-module list.
Verified: 292 passed, 4 skipped, 0 ruff, 0 ty. All 25 /ui/* routes walked
against the live app; 24x 200. /ui/documents/{id}/sources 404s via a 307 that
drops the /ui prefix, confirmed pre-existing (last touched in 6a3ee26) and
left alone as out of scope.
Co-authored-by: Copilot App <[email protected]>
306 lines
12 KiB
Python
306 lines
12 KiB
Python
"""Focused V4.2 evidence, integrity, and benchmark tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from datetime import UTC
|
|
from datetime import datetime
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
import pytest
|
|
from pydantic import JsonValue
|
|
|
|
from transcription.benchmarking import EditorialAssessment
|
|
from transcription.benchmarking import score_transcription
|
|
from transcription.config import Settings
|
|
from transcription.db.models import Document
|
|
from transcription.db.models import Job
|
|
from transcription.db.models import JobSource
|
|
from transcription.db.models import JobSourceStatus
|
|
from transcription.db.models import Source
|
|
from transcription.providers.base import ProviderError
|
|
from transcription.providers.base import TranscriptionResult
|
|
from transcription.providers.evidence import SourceEvidenceReference
|
|
from transcription.providers.openrouter import OpenRouterTranscriptionProvider
|
|
from transcription.services.documents import DocumentService
|
|
from transcription.services.evidence import EvidenceService
|
|
from transcription.services.jobs import JobDeleteBlockedError
|
|
from transcription.services.jobs import JobService
|
|
from transcription.services.sources import SourceService
|
|
from transcription.services.sources import transcribe_document_image
|
|
|
|
|
|
def _json_object(value: JsonValue) -> dict[str, JsonValue]:
|
|
"""Narrow a JSON export member to an object, asserting the export shape."""
|
|
assert isinstance(value, dict)
|
|
return value
|
|
|
|
|
|
def _json_array(value: JsonValue) -> list[JsonValue]:
|
|
"""Narrow a JSON export member to an array, asserting the export shape."""
|
|
assert isinstance(value, list)
|
|
return value
|
|
|
|
|
|
class _ChunkedAsyncStream(httpx.AsyncByteStream):
|
|
def __init__(self, chunks: list[bytes]):
|
|
self._chunks = chunks
|
|
|
|
async def __aiter__(self):
|
|
for chunk in self._chunks:
|
|
yield chunk
|
|
|
|
async def aclose(self):
|
|
return
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_openrouter_captures_exact_transport_and_secret_safe_manifest():
|
|
response_body = (
|
|
b'{"id":"gen-1","created":1,"model":"vendor/model","object":"chat.completion",'
|
|
b'"system_fingerprint":null,"choices":[{"index":0,"finish_reason":"stop",'
|
|
b'"message":{"role":"assistant","content":"Transcript"}}],'
|
|
b'"unknown_transport_field":{"retained":true}}'
|
|
)
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
assert request.headers["authorization"] == "Bearer test-key"
|
|
return httpx.Response(
|
|
200,
|
|
content=response_body,
|
|
headers={
|
|
"Content-Type": "application/json",
|
|
"X-Request-Id": "req-123",
|
|
"Set-Cookie": "must-not-persist=1",
|
|
},
|
|
request=request,
|
|
)
|
|
|
|
async_client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
provider = OpenRouterTranscriptionProvider(
|
|
settings=Settings(openrouter_api_key="test-key"),
|
|
async_client=async_client,
|
|
)
|
|
result = await provider.transcribe(
|
|
prompt_text="Literal prompt",
|
|
image_bytes=b"source-bytes",
|
|
mime_type="image/png",
|
|
source_reference=SourceEvidenceReference(
|
|
source_id=uuid4(),
|
|
digest_sha256=hashlib.sha256(b"source-bytes").hexdigest(),
|
|
byte_size=len(b"source-bytes"),
|
|
media_type="image/png",
|
|
page_number=1,
|
|
),
|
|
)
|
|
|
|
assert result.transport_evidence is not None
|
|
assert result.transport_evidence.body == response_body
|
|
assert result.transport_evidence.safe_headers == {
|
|
"content-type": "application/json",
|
|
"x-request-id": "req-123",
|
|
}
|
|
assert b'"unknown_transport_field":{"retained":true}' in result.transport_evidence.body
|
|
assert result.request_manifest is not None
|
|
serialized_manifest = json.dumps(result.request_manifest.model_dump(mode="json"))
|
|
assert "data:image/png;base64" not in serialized_manifest
|
|
assert "test-key" not in serialized_manifest
|
|
assert result.request_manifest.omitted_optional_parameters == ("temperature", "top_p")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_openrouter_captures_body_consumed_as_sdk_stream():
|
|
response_body = (
|
|
b'{"id":"gen-2","created":1,"model":"vendor/model","object":"chat.completion",'
|
|
b'"system_fingerprint":null,"choices":[{"index":0,"finish_reason":"stop",'
|
|
b'"message":{"role":"assistant","content":"Transcript"}}],'
|
|
b'"unknown_streamed_field":true}'
|
|
)
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
200,
|
|
stream=_ChunkedAsyncStream([response_body[:23], response_body[23:61], response_body[61:]]),
|
|
headers={"Content-Type": "application/json"},
|
|
request=request,
|
|
)
|
|
|
|
provider = OpenRouterTranscriptionProvider(
|
|
settings=Settings(openrouter_api_key="test-key"),
|
|
async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
|
)
|
|
result = await provider.transcribe(
|
|
prompt_text="Literal prompt",
|
|
image_bytes=b"source-bytes",
|
|
mime_type="image/png",
|
|
)
|
|
|
|
assert result.transport_evidence is not None
|
|
assert result.transport_evidence.body == response_body
|
|
assert b'"unknown_streamed_field":true' in result.transport_evidence.body
|
|
await provider.aclose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_openrouter_failure_retains_safe_response_evidence():
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
500,
|
|
content=b'{"error":{"message":"provider unavailable"}}',
|
|
headers={"Content-Type": "application/json", "Retry-After": "2", "Set-Cookie": "secret=1"},
|
|
request=request,
|
|
)
|
|
|
|
provider = OpenRouterTranscriptionProvider(
|
|
settings=Settings(openrouter_api_key="test-key"),
|
|
async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
|
)
|
|
with pytest.raises(ProviderError) as failure:
|
|
await provider.transcribe(
|
|
prompt_text="Literal prompt",
|
|
image_bytes=b"source-bytes",
|
|
mime_type="image/png",
|
|
)
|
|
|
|
evidence = failure.value.transport_evidence
|
|
assert evidence is not None
|
|
assert evidence.status_code == 500
|
|
assert evidence.body == b'{"error":{"message":"provider unavailable"}}'
|
|
assert evidence.safe_headers == {"content-type": "application/json", "retry-after": "2"}
|
|
assert str(failure.value) == "OpenRouter request failed with HTTP 500: provider unavailable"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_openrouter_does_not_reuse_prior_response_on_connection_failure():
|
|
call_count = 0
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
nonlocal call_count
|
|
call_count += 1
|
|
if call_count == 1:
|
|
return httpx.Response(500, content=b'{"error":"first"}', request=request)
|
|
raise httpx.ConnectError("connection failed", request=request)
|
|
|
|
provider = OpenRouterTranscriptionProvider(
|
|
settings=Settings(openrouter_api_key="test-key"),
|
|
async_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
|
)
|
|
with pytest.raises(ProviderError) as first_failure:
|
|
await provider.transcribe(prompt_text="First", image_bytes=b"one", mime_type="image/png")
|
|
with pytest.raises(ProviderError) as second_failure:
|
|
await provider.transcribe(prompt_text="Second", image_bytes=b"two", mime_type="image/png")
|
|
|
|
assert first_failure.value.transport_evidence is not None
|
|
assert second_failure.value.transport_evidence is not None
|
|
assert second_failure.value.transport_evidence.response_received is False
|
|
assert second_failure.value.transport_evidence.body is None
|
|
await provider.aclose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_attempts_are_append_only_and_exported_with_integrity(default_session_factory):
|
|
documents = DocumentService(session_factory=default_session_factory)
|
|
jobs = JobService(session_factory=default_session_factory)
|
|
sources = SourceService(session_factory=default_session_factory)
|
|
evidence = EvidenceService(session_factory=default_session_factory)
|
|
document = await documents.create_document(Document(name="Evidence"))
|
|
job = await jobs.create_job(Job(document_id=document.id))
|
|
source = await sources.create_source(
|
|
Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="page.png",
|
|
filename="page.png",
|
|
file_path="page.png",
|
|
file_hash="a" * 64,
|
|
file_size_bytes=10,
|
|
)
|
|
)
|
|
await sources.create_job_source(JobSource(job_id=job.id, source_id=source.id))
|
|
now = datetime.now(UTC)
|
|
|
|
await sources.update_job_source_transcription(
|
|
job_id=job.id,
|
|
source_id=source.id,
|
|
text=None,
|
|
error_detail="first failed",
|
|
error_category="external_provider_error",
|
|
failure_phase="connection",
|
|
started_at=now,
|
|
finished_at=now,
|
|
)
|
|
await sources.update_job_source_transcription(
|
|
job_id=job.id,
|
|
source_id=source.id,
|
|
text="second succeeded",
|
|
provider="openrouter",
|
|
model="vendor/model",
|
|
started_at=now,
|
|
finished_at=now,
|
|
)
|
|
|
|
attempts = await evidence.list_execution_attempts(source_id=source.id)
|
|
assert [attempt.attempt_number for attempt in attempts] == [1, 2]
|
|
assert attempts[0].status == JobSourceStatus.FAILED
|
|
assert attempts[0].error_detail == "first failed"
|
|
assert attempts[1].status == JobSourceStatus.TRANSCRIBED
|
|
assert attempts[1].raw_transcription == "second succeeded"
|
|
|
|
export = await evidence.build_evidence_export(source_id=source.id)
|
|
assert _json_object(export["source"])["digest_sha256"] == "a" * 64
|
|
assert [_json_object(item)["attempt_number"] for item in _json_array(export["attempts"])] == [1, 2]
|
|
assert "file_path" not in json.dumps(export)
|
|
with pytest.raises(JobDeleteBlockedError):
|
|
await jobs.delete_job_with_guardrails(job_id=job.id)
|
|
|
|
detail = await sources.read_source_detail(source.id)
|
|
latest_job_source = detail.latest_job_source
|
|
assert latest_job_source is not None
|
|
assert latest_job_source.execution_attempts == []
|
|
latest_attempt = await evidence.read_latest_execution_attempt(job_source_id=latest_job_source.id)
|
|
assert latest_attempt is not None
|
|
assert latest_attempt.attempt.attempt_number == 2
|
|
|
|
|
|
def test_benchmark_scoring_preserves_literal_differences():
|
|
score = score_transcription(
|
|
execution_attempt_id=uuid4(),
|
|
reference="Farm house",
|
|
candidate="farm house",
|
|
assessment=EditorialAssessment(silent_normalizations=1),
|
|
latency_ms=125,
|
|
)
|
|
assert score.character_edits == 1
|
|
assert score.word_edits == 1
|
|
assert score.assessment.silent_normalizations == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_standalone_transcription_closes_locally_created_provider(tmp_path, monkeypatch):
|
|
image_path = tmp_path / "page.png"
|
|
image_path.write_bytes(b"image")
|
|
|
|
class _Provider:
|
|
closed = False
|
|
|
|
async def transcribe(self, **kwargs):
|
|
_ = kwargs
|
|
return TranscriptionResult(text="result", provider="fixture", model="model")
|
|
|
|
async def aclose(self):
|
|
self.closed = True
|
|
|
|
provider = _Provider()
|
|
monkeypatch.setattr("transcription.services.sources.get_transcription_provider", lambda **_kwargs: provider)
|
|
|
|
result = await transcribe_document_image(
|
|
image_path,
|
|
prompt_text="Prompt",
|
|
settings=Settings(openrouter_api_key="test-key"),
|
|
)
|
|
|
|
assert result.text == "result"
|
|
assert provider.closed is True
|