generated from john/python-template
zoltan57
97b3d0fd62
Removes the duplicated registry CRUD, the hand-written not-found raises, and the three divergent media writers. Behavior is preserved: every existing Document Type and Person Role test passes unchanged, which is the primary proof for MED-11. [MED-11] Generic registry service - New services/registry.py owns RegistryService[ModelT]: list, list with counts, create with IntegrityError -> conflict mapping, read, update, delete with built-in and referenced guards, is_referenced, and label normalization/casefold keying. - DocumentTypeRegistry and PersonRoleRegistry declare only the model, error class, noun, short noun, retainer phrase, and reference columns. - DocumentService and PeopleService keep their public method names and delegate. Every user-facing message, error category, and suggestion string is reproduced verbatim; only the noun is templated. - Deleted _normalize_registry_label, _document_type_label_key, _normalize_role_label, _person_role_label_key, _document_type_is_referenced, and _person_role_is_referenced. [MED-12] Shared not-found lookup - ServiceBase._get_or_raise(model, id, *, session, error, noun, suggestion, options) loads by primary key or raises the caller's error type. - documents.py: local _get_document_or_raise deleted; replaced by _read_document and adopted at read_document, delete_document, and set_document_type, which previously bypassed the helper and hand-wrote the raise. - sources.py: 8 identical Source raises and 1 Job raise collapsed into _read_source / _get_or_raise. - jobs.py and people.py already funneled through local _not_found builders and were left alone. [MED-13][MED-01] Single media writer - New services/media_storage.py owns validate -> name -> mkdir -> write -> wrap OSError. The write runs in asyncio.to_thread, so uploads no longer block the event loop. - store_source_file, store_person_portrait, and store_homepage_image now share it and are async. Callers in store.py, people_page.py, and home_page.py await them. mkdir failures are now also translated to a domain error instead of escaping as a raw OSError. - homepage_store gains HomepageStorageError so its write reports like the others. [MED-14, partial] Service independence - New services/source_media.py owns SOURCE_MIME_TYPES, SOURCE_EXTENSIONS, lookup_source_mime_type, and supported_source_formats. - documents.py no longer imports services/sources.py. Its print projection uses the non-raising lookup and raises DocumentError, so DocumentService no longer emits a TranscriptionError. - api/v4_print.py imports the mapping from the policy module. - store.py and workflows.py still import sources.py; both are orchestration modules, which services.instructions.md:75-77 explicitly permits. - Splitting SourceService itself remains deferred to V4.7. [LOW-08] Query shape - list_sources_detail filters job_id with a JOIN on JobSource instead of loading every Source and filtering in Python. - read_source_navigation replaces the full ordered-id scan and .index() with two row-value comparisons bounded by LIMIT 1. - list_processing_artifacts gains the limit parameter its summary sibling already had. - build_evidence_export runs artifact integrity hashing and file reads through asyncio.to_thread. Tests - tests/test_service_boundaries.py: AST guard asserting no service module imports a sibling service module, plus a guard that the scan is non-empty. - tests/services/test_transcription_service.py: asserts the job_id filter emits a JOIN, and that navigation emits exactly two LIMIT queries. - tests/services/test_store.py: the two storage tests are now async. Verified: 276 passed, 4 skipped; ruff check clean.
345 lines
14 KiB
Python
345 lines
14 KiB
Python
"""Tests for SourceService revision behavior."""
|
|
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from sqlalchemy import event
|
|
|
|
from transcription.config import Settings
|
|
from transcription.db.models import Document
|
|
from transcription.db.models import Job
|
|
from transcription.db.models import JobSource
|
|
from transcription.db.models import JobSourceStatus
|
|
from transcription.db.models import JobStatus
|
|
from transcription.db.models import Source
|
|
from transcription.services.documents import DocumentService
|
|
from transcription.services.jobs import JobService
|
|
from transcription.services.sources import SourceDeleteBlockedError
|
|
from transcription.services.sources import SourceService
|
|
from transcription.services.sources import TranscriptionNotFoundError
|
|
|
|
|
|
@pytest.mark.integration
|
|
class TestSourceServiceRevisionUpsert:
|
|
"""Verify page-level source revision semantics."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_upsert_revision_creates_new_revision(self, default_session_factory):
|
|
documents = DocumentService(session_factory=default_session_factory)
|
|
jobs = JobService(session_factory=default_session_factory)
|
|
transcriptions = SourceService(session_factory=default_session_factory)
|
|
|
|
document = Document(id=uuid4(), name="revision-create")
|
|
await documents.create_document(document=document)
|
|
|
|
job = Job(document_id=document.id, status=JobStatus.TRANSCRIBED)
|
|
await jobs.create_job(job=job)
|
|
|
|
source = Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="source.jpg",
|
|
filename="source.jpg",
|
|
file_path="uploads/source.jpg",
|
|
file_hash="1" * 64,
|
|
file_size_bytes=1,
|
|
)
|
|
async with transcriptions._session_scope() as session:
|
|
session.add(source)
|
|
await session.flush()
|
|
session.add(JobSource(job_id=job.id, source_id=source.id, status=JobSourceStatus.PENDING))
|
|
await session.commit()
|
|
await session.refresh(source)
|
|
|
|
revision = await transcriptions.upsert_revision_for_source(source_id=source.id, text="User revision")
|
|
fetched = await transcriptions.read_revision_by_source(source.id)
|
|
|
|
assert revision.id == source.id
|
|
assert revision.revised_text == "User revision"
|
|
assert fetched is not None
|
|
assert fetched.id == source.id
|
|
assert fetched.revised_text == "User revision"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_upsert_revision_updates_existing_single_revision(self, default_session_factory):
|
|
documents = DocumentService(session_factory=default_session_factory)
|
|
jobs = JobService(session_factory=default_session_factory)
|
|
transcriptions = SourceService(session_factory=default_session_factory)
|
|
|
|
document = Document(id=uuid4(), name="revision-update")
|
|
await documents.create_document(document=document)
|
|
|
|
job = Job(document_id=document.id, status=JobStatus.TRANSCRIBED)
|
|
await jobs.create_job(job=job)
|
|
|
|
source = Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="source.jpg",
|
|
filename="source.jpg",
|
|
file_path="uploads/source.jpg",
|
|
file_hash="2" * 64,
|
|
file_size_bytes=1,
|
|
)
|
|
async with transcriptions._session_scope() as session:
|
|
session.add(source)
|
|
await session.flush()
|
|
session.add(JobSource(job_id=job.id, source_id=source.id, status=JobSourceStatus.PENDING))
|
|
await session.commit()
|
|
await session.refresh(source)
|
|
|
|
first = await transcriptions.upsert_revision_for_source(source_id=source.id, text="Revision v1")
|
|
second = await transcriptions.upsert_revision_for_source(source_id=source.id, text="Revision v2")
|
|
revisions = await transcriptions.list_revisions_by_job(job.id)
|
|
|
|
assert first.id == second.id
|
|
assert second.revised_text == "Revision v2"
|
|
assert len(revisions) == 1
|
|
assert revisions[0].id == first.id
|
|
assert revisions[0].revised_text == "Revision v2"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_source_from_job_context_removes_source_and_single_link(
|
|
self, default_session_factory, tmp_path
|
|
):
|
|
documents = DocumentService(session_factory=default_session_factory)
|
|
jobs = JobService(session_factory=default_session_factory)
|
|
settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
|
|
transcriptions = SourceService(session_factory=default_session_factory, settings=settings)
|
|
|
|
document = Document(id=uuid4(), name="delete-source-success")
|
|
await documents.create_document(document=document)
|
|
|
|
job = Job(document_id=document.id, status=JobStatus.QUEUED)
|
|
await jobs.create_job(job=job)
|
|
|
|
stored_path = tmp_path / "documents" / str(document.id) / "delete.jpg"
|
|
stored_path.parent.mkdir(parents=True, exist_ok=True)
|
|
stored_path.write_bytes(b"data")
|
|
|
|
source = Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="delete.jpg",
|
|
filename="delete.jpg",
|
|
file_path=str(stored_path),
|
|
file_hash="3" * 64,
|
|
file_size_bytes=4,
|
|
)
|
|
async with transcriptions._session_scope() as session:
|
|
session.add(source)
|
|
await session.flush()
|
|
session.add(JobSource(job_id=job.id, source_id=source.id, status=JobSourceStatus.PENDING))
|
|
await session.commit()
|
|
await session.refresh(source)
|
|
|
|
await transcriptions.delete_source_from_job_context(job_id=job.id, source_id=source.id)
|
|
|
|
with pytest.raises(TranscriptionNotFoundError):
|
|
await transcriptions.read_source(source.id)
|
|
assert not stored_path.exists()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_source_from_job_context_blocks_when_other_job_links_exist(self, default_session_factory):
|
|
documents = DocumentService(session_factory=default_session_factory)
|
|
jobs = JobService(session_factory=default_session_factory)
|
|
transcriptions = SourceService(session_factory=default_session_factory)
|
|
|
|
document = Document(id=uuid4(), name="delete-source-blocked")
|
|
await documents.create_document(document=document)
|
|
|
|
job_one = Job(document_id=document.id, status=JobStatus.QUEUED)
|
|
job_two = Job(document_id=document.id, status=JobStatus.QUEUED)
|
|
await jobs.create_job(job=job_one)
|
|
await jobs.create_job(job=job_two)
|
|
|
|
source = Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="shared.jpg",
|
|
filename="shared.jpg",
|
|
file_path="uploads/shared.jpg",
|
|
file_hash="4" * 64,
|
|
file_size_bytes=1,
|
|
)
|
|
async with transcriptions._session_scope() as session:
|
|
session.add(source)
|
|
await session.flush()
|
|
session.add(JobSource(job_id=job_one.id, source_id=source.id, status=JobSourceStatus.PENDING))
|
|
session.add(JobSource(job_id=job_two.id, source_id=source.id, status=JobSourceStatus.PENDING))
|
|
await session.commit()
|
|
await session.refresh(source)
|
|
|
|
with pytest.raises(SourceDeleteBlockedError):
|
|
await transcriptions.delete_source_from_job_context(job_id=job_one.id, source_id=source.id)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_unlinked_source_succeeds(self, default_session_factory, tmp_path):
|
|
documents = DocumentService(session_factory=default_session_factory)
|
|
settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
|
|
transcriptions = SourceService(session_factory=default_session_factory, settings=settings)
|
|
|
|
document = Document(id=uuid4(), name="delete-unlinked-source")
|
|
await documents.create_document(document=document)
|
|
|
|
stored_path = tmp_path / "documents" / str(document.id) / "orphan.jpg"
|
|
stored_path.parent.mkdir(parents=True, exist_ok=True)
|
|
stored_path.write_bytes(b"data")
|
|
|
|
source = Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="orphan.jpg",
|
|
filename="orphan.jpg",
|
|
file_path=str(stored_path),
|
|
file_hash="5" * 64,
|
|
file_size_bytes=4,
|
|
)
|
|
await transcriptions.create_source(source=source)
|
|
|
|
await transcriptions.delete_unlinked_source(source_id=source.id)
|
|
|
|
with pytest.raises(TranscriptionNotFoundError):
|
|
await transcriptions.read_source(source.id)
|
|
assert not stored_path.exists()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_unlinked_source_blocks_when_linked(self, default_session_factory):
|
|
documents = DocumentService(session_factory=default_session_factory)
|
|
jobs = JobService(session_factory=default_session_factory)
|
|
transcriptions = SourceService(session_factory=default_session_factory)
|
|
|
|
document = Document(id=uuid4(), name="delete-unlinked-blocked")
|
|
await documents.create_document(document=document)
|
|
|
|
job = Job(document_id=document.id, status=JobStatus.QUEUED)
|
|
await jobs.create_job(job=job)
|
|
|
|
source = Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="linked.jpg",
|
|
filename="linked.jpg",
|
|
file_path="uploads/linked.jpg",
|
|
file_hash="6" * 64,
|
|
file_size_bytes=1,
|
|
)
|
|
async with transcriptions._session_scope() as session:
|
|
session.add(source)
|
|
await session.flush()
|
|
session.add(JobSource(job_id=job.id, source_id=source.id, status=JobSourceStatus.PENDING))
|
|
await session.commit()
|
|
await session.refresh(source)
|
|
|
|
with pytest.raises(SourceDeleteBlockedError):
|
|
await transcriptions.delete_unlinked_source(source_id=source.id)
|
|
|
|
|
|
@pytest.mark.integration
|
|
class TestSourceServiceQueryShape:
|
|
"""LOW-08: reads must filter and bound in SQL, not in Python."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_list_sources_detail_filters_job_id_with_a_join(self, default_session_factory):
|
|
documents = DocumentService(session_factory=default_session_factory)
|
|
jobs = JobService(session_factory=default_session_factory)
|
|
transcriptions = SourceService(session_factory=default_session_factory)
|
|
|
|
document = Document(id=uuid4(), name="join-filter")
|
|
await documents.create_document(document=document)
|
|
job = Job(document_id=document.id, status=JobStatus.QUEUED)
|
|
other_job = Job(document_id=document.id, status=JobStatus.QUEUED)
|
|
await jobs.create_job(job=job)
|
|
await jobs.create_job(job=other_job)
|
|
|
|
linked = Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="linked.jpg",
|
|
filename="linked.jpg",
|
|
file_path="uploads/linked.jpg",
|
|
file_hash="7" * 64,
|
|
file_size_bytes=1,
|
|
)
|
|
unlinked = Source(
|
|
document_id=document.id,
|
|
page_number=2,
|
|
upload_name="unlinked.jpg",
|
|
filename="unlinked.jpg",
|
|
file_path="uploads/unlinked.jpg",
|
|
file_hash="8" * 64,
|
|
file_size_bytes=1,
|
|
)
|
|
async with transcriptions._session_scope() as session:
|
|
session.add_all((linked, unlinked))
|
|
await session.flush()
|
|
session.add(JobSource(job_id=job.id, source_id=linked.id, status=JobSourceStatus.PENDING))
|
|
session.add(JobSource(job_id=other_job.id, source_id=unlinked.id, status=JobSourceStatus.PENDING))
|
|
await session.commit()
|
|
|
|
statements: list[str] = []
|
|
|
|
async with transcriptions._session_scope() as session:
|
|
bind = session.get_bind()
|
|
|
|
def capture(_conn, _cursor, statement, *_rest):
|
|
statements.append(statement)
|
|
|
|
event.listen(bind, "before_cursor_execute", capture)
|
|
try:
|
|
sources = await transcriptions.list_sources_detail(job_id=job.id, session=session)
|
|
finally:
|
|
event.remove(bind, "before_cursor_execute", capture)
|
|
|
|
assert [source.id for source in sources] == [linked.id]
|
|
|
|
primary = next(item for item in statements if item.lstrip().upper().startswith("SELECT"))
|
|
assert "JOIN" in primary.upper()
|
|
assert "JOBSOURCE" in primary.upper().replace("_", "")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_read_source_navigation_does_not_scan_every_sibling(self, default_session_factory):
|
|
documents = DocumentService(session_factory=default_session_factory)
|
|
transcriptions = SourceService(session_factory=default_session_factory)
|
|
|
|
document = Document(id=uuid4(), name="navigation-bounds")
|
|
await documents.create_document(document=document)
|
|
|
|
pages = [
|
|
Source(
|
|
document_id=document.id,
|
|
page_number=page_number,
|
|
upload_name=f"page-{page_number}.jpg",
|
|
filename=f"page-{page_number}.jpg",
|
|
file_path=f"uploads/page-{page_number}.jpg",
|
|
file_hash=str(page_number) * 64,
|
|
file_size_bytes=1,
|
|
)
|
|
for page_number in range(1, 5)
|
|
]
|
|
async with transcriptions._session_scope() as session:
|
|
session.add_all(pages)
|
|
await session.commit()
|
|
for page in pages:
|
|
await session.refresh(page)
|
|
|
|
statements: list[str] = []
|
|
|
|
async with transcriptions._session_scope() as session:
|
|
bind = session.get_bind()
|
|
|
|
def capture(_conn, _cursor, statement, *_rest):
|
|
statements.append(statement)
|
|
|
|
event.listen(bind, "before_cursor_execute", capture)
|
|
try:
|
|
navigation = await transcriptions.read_source_navigation(pages[1].id, session=session)
|
|
finally:
|
|
event.remove(bind, "before_cursor_execute", capture)
|
|
|
|
assert navigation.previous_id == pages[0].id
|
|
assert navigation.next_id == pages[2].id
|
|
|
|
adjacency = [item for item in statements if "LIMIT" in item.upper()]
|
|
assert len(adjacency) == 2, statements
|