bbchops/transcription
1
0
Fork 0
generated from john/python-template
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

V4.7 Phase 2: Evidence Model Simplification (part 2)

Browse Source
This commit is contained in:
zoltan57
2026-08-18 15:31:33 -05:00
parent 7285a87dfb
commit 11097b9cfe
15 changed files with 312 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration/test_pipeline_flow.py
+29 -15
View File
@@ -6,9 +6,12 @@ from uuid import uuid4
import pytest
from PIL import Image
from sqlmodel import col
from sqlmodel import select
from transcription.config import Settings
from transcription.db.models import Document
from transcription.db.models import ExecutionAttempt
from transcription.db.models import JobSourceStatus
from transcription.db.models import JobStatus
from transcription.providers.base import ProviderUsage
@@ -20,6 +23,15 @@ from transcription.services.store import create_job_for_document
from transcription.services.workflows import advance_job
async def _attempts_for_job(session, job) -> list[ExecutionAttempt]:
"""Load execution attempts for a job; V4.7 moved evidence off JobSource."""
job_source_ids = [job_source.id for job_source in job.job_sources]
result = await session.exec(
select(ExecutionAttempt).where(col(ExecutionAttempt.job_source_id).in_(job_source_ids))
)
return list(result.all())
def _jpeg_bytes(color: str = "white") -> bytes:
output = io.BytesIO()
Image.new("RGB", (2, 2), color=color).save(output, format="JPEG")
@@ -116,21 +128,24 @@ class TestPipelineSuccessFlow:
assert processed is True
assert job is not None
assert job.status == JobStatus.TRANSCRIBED
assert any(job_source.raw_transcription == "Pipeline transcript" for job_source in job.job_sources)
attempts = await _attempts_for_job(async_session, job)
assert any(attempt.raw_transcription == "Pipeline transcript" for attempt in attempts)
assert job.prompt_name == "transcribe_document.md"
assert job.user_prompt is not None
assert job.temperature == 0.2
assert job.top_p == 0.85
assert any(
job_source.ai_metadata == {"finish_reason": "stop", "usage": {"total_tokens": 42}}
for job_source in job.job_sources
attempt.normalized_metadata is not None
and attempt.normalized_metadata["finish_reason"] == "stop"
and attempt.normalized_metadata["usage"] == {"total_tokens": 42}
for attempt in attempts
)
assert any(
job_source.raw_api_response
attempt.sdk_response_snapshot
== {"id": "resp_123", "choices": [{"message": {"content": "Pipeline transcript"}}]}
for job_source in job.job_sources
for attempt in attempts
)
assert all(job_source.error_detail is None for job_source in job.job_sources)
assert all(attempt.error_detail is None for attempt in attempts)
@pytest.mark.asyncio
async def test_worker_transcribes_all_sources_for_multi_page_job(
@@ -191,7 +206,8 @@ class TestPipelineSuccessFlow:
assert job.status == JobStatus.TRANSCRIBED
assert len(job.job_sources) == 3
assert all(job_source.status == JobSourceStatus.TRANSCRIBED for job_source in job.job_sources)
assert all(job_source.raw_transcription for job_source in job.job_sources)
attempts = await _attempts_for_job(async_session, job)
assert all(attempt.raw_transcription for attempt in attempts)
assert all(
job_source.source is not None and job_source.source.raw_transcription for job_source in job.job_sources
)
@@ -261,7 +277,8 @@ class TestPipelineSuccessFlow:
assert len(job.job_sources) == 2
statuses = {job_source.status for job_source in job.job_sources}
assert statuses == {JobSourceStatus.TRANSCRIBED, JobSourceStatus.FAILED}
assert any(job_source.error_detail is not None for job_source in job.job_sources)
attempts = await _attempts_for_job(async_session, job)
assert any(attempt.error_detail is not None for attempt in attempts)
@pytest.mark.asyncio
async def test_worker_skips_already_transcribed_sources_on_resubmit(
@@ -293,9 +310,7 @@ class TestPipelineSuccessFlow:
page_two = next(js for js in job.job_sources if js.source is not None and js.source.page_number == 2)
page_one.status = JobSourceStatus.TRANSCRIBED
page_one.raw_transcription = "existing transcript"
page_two.status = JobSourceStatus.PENDING
page_two.raw_transcription = None
await services.sources.update_job_source(job_source=page_one, session=async_session)
await services.sources.update_job_source(job_source=page_two, session=async_session)
await services.jobs.update_job_state(job_id=job.id, status=JobStatus.QUEUED, session=async_session)
@@ -387,11 +402,10 @@ class TestPipelineFailureFlow:
assert processed is True
assert job is not None
assert job.status == JobStatus.FAILED
assert all(job_source.raw_transcription is None for job_source in job.job_sources)
assert any(job_source.error_detail is not None for job_source in job.job_sources)
error_detail = next(
job_source.error_detail for job_source in job.job_sources if job_source.error_detail is not None
)
attempts = await _attempts_for_job(async_session, job)
assert all(attempt.raw_transcription is None for attempt in attempts)
assert any(attempt.error_detail is not None for attempt in attempts)
error_detail = next(attempt.error_detail for attempt in attempts if attempt.error_detail is not None)
assert "pipeline provider failure" in error_detail
assert "[internal_unexpected_error]" in error_detail
assert "error_id=" in error_detail
tests/services/test_job_service.py
+42 -9
View File
@@ -342,7 +342,6 @@ class TestJobService:
job_id=job.id,
source_id=source_one.id,
status=JobSourceStatus.TRANSCRIBED,
raw_transcription="done",
)
)
session.add(
@@ -360,9 +359,8 @@ class TestJobService:
refreshed = await job_service.read_job(job_id=job.id)
statuses = {item.status for item in refreshed.job_sources}
assert JobSourceStatus.TRANSCRIBED in statuses
assert JobSourceStatus.FAILED in statuses
pending_entry = next(item for item in refreshed.job_sources if item.status == JobSourceStatus.FAILED)
assert pending_entry.error_detail == "Cancelled by user"
assert JobSourceStatus.CANCELLED in statuses
assert JobSourceStatus.FAILED not in statuses
@pytest.mark.asyncio
async def test_resubmit_failed_sources_resets_only_failed(
@@ -406,8 +404,6 @@ class TestJobService:
job_id=job.id,
source_id=source_one.id,
status=JobSourceStatus.FAILED,
raw_transcription=None,
error_detail="prior error",
)
)
session.add(
@@ -415,7 +411,6 @@ class TestJobService:
job_id=job.id,
source_id=source_two.id,
status=JobSourceStatus.TRANSCRIBED,
raw_transcription="done text",
)
)
await session.commit()
@@ -433,7 +428,6 @@ class TestJobService:
item for item in refreshed.job_sources if item.source is not None and item.source.page_number == 2
)
assert failed_entry.status == JobSourceStatus.PENDING
assert failed_entry.error_detail is None
assert failed_entry.source is not None
assert failed_entry.source.raw_transcription == "existing text"
assert transcribed_entry.status == JobSourceStatus.TRANSCRIBED
@@ -486,7 +480,6 @@ class TestJobService:
job_id=job.id,
source_id=source_two.id,
status=JobSourceStatus.TRANSCRIBED,
raw_transcription="done text",
)
)
await session.commit()
@@ -494,6 +487,46 @@ class TestJobService:
with pytest.raises(JobResubmitBlockedError):
await job_service.resubmit_failed_sources(job_id=job.id)
@pytest.mark.asyncio
async def test_resubmit_failed_sources_includes_cancelled(
self,
job_service: JobService,
document_service: DocumentService,
):
"""Cancel is recoverable: cancelled pages are re-attempted on resubmit."""
document = Document(id=uuid4(), name="resubmit-cancelled-doc")
await document_service.create_document(document=document)
job = Job(document_id=document.id, status=JobStatus.FAILED)
await job_service.create_job(job=job)
async with job_service._session_scope() as session:
source = Source(
document_id=document.id,
page_number=1,
upload_name="resubmit-cancelled.jpg",
filename="stored-resubmit-cancelled.jpg",
file_path="/uploads/stored-resubmit-cancelled.jpg",
file_hash="3" * 64,
file_size_bytes=1,
)
session.add(source)
await session.flush()
session.add(
JobSource(
job_id=job.id,
source_id=source.id,
status=JobSourceStatus.CANCELLED,
)
)
await session.commit()
assert await job_service.resubmit_failed_sources(job_id=job.id) == 1
refreshed = await job_service.read_job(job_id=job.id)
assert refreshed.status == JobStatus.QUEUED
assert refreshed.job_sources[0].status == JobSourceStatus.PENDING
@pytest.mark.asyncio
async def test_resubmit_failed_sources_blocks_when_processing(
self,
tests/services/test_v2_crud.py
+7 -3
View File
@@ -322,6 +322,10 @@ async def test_update_job_source_transcription_persists_provider_json_payloads(d
stored_rows = await transcriptions.list_job_sources(job_id=job.id)
assert len(stored_rows) == 1
assert stored_rows[0].raw_transcription == "provider transcript"
assert stored_rows[0].ai_metadata == metadata
assert stored_rows[0].raw_api_response == raw_payload
assert stored_rows[0].status == JobSourceStatus.TRANSCRIBED
attempt = await transcriptions.read_latest_execution_attempt(job_source_id=stored_rows[0].id)
assert attempt is not None
assert attempt.attempt.raw_transcription == "provider transcript"
assert attempt.attempt.normalized_metadata == metadata
assert attempt.attempt.sdk_response_snapshot == raw_payload
tests/services/test_workflows_reliability.py
+15 -3
View File
@@ -5,9 +5,12 @@ from pathlib import Path
from uuid import uuid4
import pytest
from sqlmodel import col
from sqlmodel import select
from transcription.config import Settings
from transcription.db.models import Document
from transcription.db.models import ExecutionAttempt
from transcription.db.models import Job
from transcription.db.models import JobSource
from transcription.db.models import JobSourceStatus
@@ -87,9 +90,18 @@ class TestWorkflowReliability:
assert result is not None
assert result.status == JobStatus.FAILED
assert result.error_detail is not None
assert "timed out" in result.error_detail.lower()
assert "20.0s" in result.error_detail
async with services.jobs._session_scope() as session:
attempts = (
await session.execute(
select(ExecutionAttempt).where(
col(ExecutionAttempt.job_source_id).in_([js.id for js in result.job_sources])
)
)
).scalars().all()
error_detail = next(attempt.error_detail for attempt in attempts if attempt.error_detail is not None)
assert "timed out" in error_detail.lower()
assert "20.0s" in error_detail
@pytest.mark.asyncio
async def test_completed_page_is_committed_before_next_provider_call_finishes(
tests/test_models.py
+4 -11
View File
@@ -211,24 +211,17 @@ class TestPersonAndDocumentPersonModel:
class TestJobSourceModel:
def test_job_source_persists_json_payloads(self, session):
def test_job_source_is_a_queue_row_not_an_evidence_row(self, session):
"""V4.7: job_source carries only queue state; evidence lives on execution_attempt."""
document = _persist_document(session)
job = _persist_job(session, document)
source = _persist_source(session, document)
job_source = _persist_job_source(
session,
job,
source,
raw_transcription="Page transcript",
ai_metadata={"confidence": 0.91, "boxes": [{"x": 1, "y": 2}]},
raw_api_response={"provider": "test"},
)
job_source = _persist_job_source(session, job, source)
fetched = session.get(JobSource, job_source.id)
assert fetched is not None
assert fetched.status == JobSourceStatus.PENDING
assert fetched.ai_metadata == {"confidence": 0.91, "boxes": [{"x": 1, "y": 2}]}
assert fetched.raw_api_response == {"provider": "test"}
assert set(JobSource.model_fields) == {"id", "job_id", "source_id", "status"}
class TestRelationships:
tests/ui/conftest.py
+20 -6
View File
@@ -23,6 +23,7 @@ from transcription.db import session as db_session_module
from transcription.db import session_scope
from transcription.db.models import Document
from transcription.db.models import DocumentPerson
from transcription.db.models import ExecutionAttempt
from transcription.db.models import Job
from transcription.db.models import JobSource
from transcription.db.models import JobSourceStatus
@@ -131,17 +132,30 @@ async def seed_job(app_client: tuple[FastAPI, TestClient]) -> Callable[..., Awai
await session.flush()
if transcription_text is not None or error_detail is not None:
outcome = JobSourceStatus.TRANSCRIBED if transcription_text is not None else JobSourceStatus.FAILED
job_source = JobSource(job_id=job.id, source_id=source.id, status=outcome)
session.add(job_source)
await session.flush()
# V4.7: evidence lives on execution_attempt, not job_source.
executed_at = datetime.now(UTC)
session.add(
JobSource(
ExecutionAttempt(
job_source_id=job_source.id,
job_id=job.id,
source_id=source.id,
status=(
JobSourceStatus.TRANSCRIBED if transcription_text is not None else JobSourceStatus.FAILED
),
attempt_number=1,
status=outcome,
provider="openrouter",
model="google/gemini-2.5-flash",
response_received=transcription_text is not None,
sdk_response_snapshot=raw_api_response,
normalized_metadata=ai_metadata,
raw_transcription=transcription_text,
error_detail=error_detail,
ai_metadata=ai_metadata,
raw_api_response=raw_api_response,
started_at=executed_at,
finished_at=executed_at,
duration_ms=0,
)
)
tests/ui/test_jobs_page.py
+1 -1
View File
@@ -125,7 +125,7 @@ class TestJobsPageRendering:
assert response.status_code == 200
assert "Resubmit Job" in response.text
assert "Failed Sources:" in response.text
assert "Resubmittable Sources:" in response.text
assert "Resubmit now" in response.text
@pytest.mark.asyncio
tests/ui/test_sources_page.py
+5 -3
View File
@@ -7,9 +7,11 @@ import pytest
from sqlmodel import select
from transcription.db import session_scope
from transcription.db.loading import orm_attribute
from transcription.db.loading import selectinload
from transcription.db.models import Document
from transcription.db.models import Job
from transcription.db.models import JobSource
from transcription.db.models import JobSourceStatus
from transcription.db.models import JobStatus
from transcription.db.models import Source
@@ -56,7 +58,7 @@ class TestSourceModelProperties:
select(Source)
.options(
selectinload(Source.document),
selectinload(Source.job_sources),
selectinload(Source.job_sources).selectinload(orm_attribute(JobSource.execution_attempts)),
)
.where(Source.document_id == job.document_id)
)
@@ -218,8 +220,8 @@ class TestSourcesPageRendering:
assert "Save revision" in response.text
assert "Previous Page" in response.text
assert "Next Page" in response.text
assert "AI Metadata" in response.text
assert "Raw API Response" in response.text
assert "Normalized Metadata" in response.text
assert "OpenRouter SDK Response Snapshot" in response.text
assert "finish_reason" in response.text
assert "response-123" in response.text