generated from john/python-template
zoltan57andCopilot App
110f40a28b
Review log [55]. Three historical local_timeout rows recorded 0.4-2.0s more
than the configured budget because the measurement window opened before the
provider call.
The plan named two causes, and both were already gone. Diffed against
f86c0ff~1: at V4.6 the window held resolve_provider_input (async;
normalization + artifact write + DB work) and a session.commit(). Phase 1
deleted both. What remains between the clock and the wait_for is
build_provider_input, now pure field copying because normalization moved to
ingest and file_hash is already stored: 6.2 us per call, zero awaits, so it
cannot yield to the event loop.
A third cause was still there and is not in the plan. The regression test
below measured 890ms where ~200ms was expected. services.sources.provider is
a lazy property that appears as an argument expression to _call_transcriber,
so it is evaluated after the clock starts but before wait_for begins timing.
Constructing OpenRouterTranscriptionProvider costs 475ms on first access and
0.001ms after, so the first attempt of every worker process booked half a
second of HTTP client construction as provider latency. That plausibly
accounts for the low end of the historical overshoot.
workflows.py
- Re-capture monotonic_started_at immediately before the wait_for, reusing
the same variable. The pre-loop assignment stays as the fallback: binding
a new name inside the try would leave the general-exception handler
referencing an unbound variable when build_provider_input raises. All
three duration write sites (success, TimeoutError, general failure) then
measure the correct window with no further change.
- Hoist the provider property above the per-source loop. It is
loop-invariant, so this also removes the repeated lookup from the two
evidence-capture sites.
tests/services/test_workflows_reliability.py
test_timeout_duration_excludes_pre_call_setup simulates 400ms of blocking
setup against a 200ms budget and asserts the recorded duration sits near
the budget and well clear of budget+setup. Confirmed to fail on the pre-fix
code (assert 625 < 540) and pass after, so it guards behaviour rather than
restating it. This is the plan's verification criterion as a test.
ui/pages/sources_page.py
_format_duration renders >=1s as "27.6 s" and below that as "612 ms",
replacing the raw "27612 ms". No test asserted the old format.
Plan task 3 (record preprocessing as its own value) declined and logged as a
deviation: after Phase 1 there is no preprocessing left to record, and a
preprocessing_ms column to measure 6 us of attribute copying is complexity
without a reader.
Verified: 293 passed, 4 skipped, 0 ruff, 0 ty.
Co-authored-by: Copilot App <[email protected]>
236 lines
9.4 KiB
Python
236 lines
9.4 KiB
Python
"""Reliability tests for worker workflow timeout behavior."""
|
|
|
|
import asyncio
|
|
import time
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from sqlmodel import col
|
|
from sqlmodel import select
|
|
|
|
from transcription.config import Settings
|
|
from transcription.db.models import Document
|
|
from transcription.db.models import ExecutionAttempt
|
|
from transcription.db.models import Job
|
|
from transcription.db.models import JobSource
|
|
from transcription.db.models import JobSourceStatus
|
|
from transcription.db.models import JobStatus
|
|
from transcription.db.models import Source
|
|
from transcription.providers.base import TranscriptionResult
|
|
from transcription.services import ServiceBundle
|
|
from transcription.services import workflows as workflows_module
|
|
from transcription.services.workflows import process_queued_job
|
|
|
|
|
|
@pytest.mark.integration
|
|
class TestWorkflowReliability:
|
|
"""Verify timeout and terminal-state reliability behavior."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_process_queued_job_timeout_marks_job_failed(self, default_session_factory, monkeypatch):
|
|
"""Provider timeout transitions a queued job to failed with error detail."""
|
|
services = ServiceBundle()
|
|
object.__setattr__(
|
|
services,
|
|
"documents",
|
|
services.documents.__class__(session_factory=default_session_factory),
|
|
)
|
|
object.__setattr__(
|
|
services,
|
|
"jobs",
|
|
services.jobs.__class__(session_factory=default_session_factory),
|
|
)
|
|
object.__setattr__(
|
|
services,
|
|
"sources",
|
|
services.sources.__class__(session_factory=default_session_factory),
|
|
)
|
|
|
|
async with services.jobs._session_scope() as session:
|
|
document = Document(id=uuid4(), name="timeout-doc")
|
|
session.add(document)
|
|
await session.flush()
|
|
|
|
job = Job(document_id=document.id, status=JobStatus.QUEUED)
|
|
session.add(job)
|
|
await session.flush()
|
|
|
|
source = Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="timeout.jpg",
|
|
filename="timeout.jpg",
|
|
file_path=str(Path("tests/fixtures/images/real/Book Two - page 02.jpg")),
|
|
file_hash="c" * 64,
|
|
file_size_bytes=1,
|
|
)
|
|
session.add(source)
|
|
await session.flush()
|
|
session.add(JobSource(job_id=job.id, source_id=source.id, status=JobSourceStatus.PENDING))
|
|
await session.commit()
|
|
|
|
loaded = await services.jobs.read_job(job_id=job.id, session=session)
|
|
|
|
async def _never_returns(
|
|
image_path,
|
|
*,
|
|
prompt_name="transcribe_document.md",
|
|
prompt_text=None,
|
|
temperature=None,
|
|
top_p=None,
|
|
settings=None,
|
|
provider=None,
|
|
):
|
|
_ = (image_path, prompt_name, prompt_text, temperature, top_p, settings, provider)
|
|
raise TimeoutError("simulated provider timeout")
|
|
|
|
monkeypatch.setattr("transcription.services.workflows.transcribe_document_image", _never_returns)
|
|
|
|
timeout_settings = Settings(openrouter_api_key="test-key", worker_provider_timeout_seconds=20.0)
|
|
result = await process_queued_job(job=loaded, services=services, settings=timeout_settings)
|
|
|
|
assert result is not None
|
|
assert result.status == JobStatus.FAILED
|
|
|
|
async with services.jobs._session_scope() as session:
|
|
attempts = (
|
|
await session.execute(
|
|
select(ExecutionAttempt).where(
|
|
col(ExecutionAttempt.job_source_id).in_([js.id for js in result.job_sources])
|
|
)
|
|
)
|
|
).scalars().all()
|
|
error_detail = next(attempt.error_detail for attempt in attempts if attempt.error_detail is not None)
|
|
assert "timed out" in error_detail.lower()
|
|
assert "20.0s" in error_detail
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_timeout_duration_excludes_pre_call_setup(self, default_session_factory, monkeypatch):
|
|
"""duration_ms covers only the provider call, not the setup preceding it.
|
|
|
|
Regression guard for review log [55]: three historical ``local_timeout`` rows
|
|
recorded 0.4-2.0 s more than the configured budget because the measurement
|
|
window opened before payload resolution. Blocking setup is simulated here so
|
|
the assertion fails if that window ever reopens.
|
|
"""
|
|
services = ServiceBundle.from_session_factory(default_session_factory)
|
|
async with services.jobs._session_scope() as session:
|
|
document = Document(id=uuid4(), name="window-doc")
|
|
session.add(document)
|
|
await session.flush()
|
|
job = Job(document_id=document.id, status=JobStatus.QUEUED)
|
|
session.add(job)
|
|
await session.flush()
|
|
source = Source(
|
|
document_id=document.id,
|
|
page_number=1,
|
|
upload_name="window.jpg",
|
|
filename="window.jpg",
|
|
file_path=str(Path("tests/fixtures/images/real/Book Two - page 02.jpg")),
|
|
file_hash="d" * 64,
|
|
file_size_bytes=1,
|
|
)
|
|
session.add(source)
|
|
await session.flush()
|
|
session.add(JobSource(job_id=job.id, source_id=source.id, status=JobSourceStatus.PENDING))
|
|
await session.commit()
|
|
loaded = await services.jobs.read_job(job_id=job.id, session=session)
|
|
|
|
setup_seconds = 0.40
|
|
budget_seconds = 0.20
|
|
real_build = workflows_module.build_provider_input
|
|
|
|
def _slow_build(source_arg):
|
|
time.sleep(setup_seconds)
|
|
return real_build(source_arg)
|
|
|
|
async def _never_returns(*args, **kwargs):
|
|
_ = (args, kwargs)
|
|
await asyncio.sleep(budget_seconds * 20)
|
|
|
|
monkeypatch.setattr("transcription.services.workflows.build_provider_input", _slow_build)
|
|
monkeypatch.setattr("transcription.services.workflows.transcribe_document_image", _never_returns)
|
|
|
|
result = await process_queued_job(
|
|
job=loaded,
|
|
services=services,
|
|
settings=Settings(openrouter_api_key="test-key", worker_provider_timeout_seconds=budget_seconds),
|
|
)
|
|
assert result is not None
|
|
assert result.status == JobStatus.FAILED
|
|
|
|
async with services.jobs._session_scope() as session:
|
|
attempts = (
|
|
(
|
|
await session.exec(
|
|
select(ExecutionAttempt).where(
|
|
col(ExecutionAttempt.job_source_id).in_([js.id for js in result.job_sources])
|
|
)
|
|
)
|
|
)
|
|
.all()
|
|
)
|
|
assert len(attempts) == 1
|
|
duration_ms = attempts[0].duration_ms
|
|
|
|
# At or just above the budget, and well clear of budget + setup.
|
|
assert duration_ms >= int(budget_seconds * 1000 * 0.9)
|
|
assert duration_ms < int((budget_seconds + setup_seconds) * 1000 * 0.9)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_completed_page_is_committed_before_next_provider_call_finishes(
|
|
self,
|
|
default_session_factory,
|
|
monkeypatch,
|
|
):
|
|
services = ServiceBundle.from_session_factory(default_session_factory)
|
|
async with services.jobs._session_scope() as session:
|
|
document = Document(id=uuid4(), name="durability-doc")
|
|
session.add(document)
|
|
await session.flush()
|
|
job = Job(document_id=document.id, status=JobStatus.QUEUED)
|
|
session.add(job)
|
|
await session.flush()
|
|
for page_number in (1, 2):
|
|
source = Source(
|
|
document_id=document.id,
|
|
page_number=page_number,
|
|
upload_name=f"page-{page_number}.jpg",
|
|
filename=f"page-{page_number}.jpg",
|
|
file_path=str(Path("tests/fixtures/images/real/Book Two - page 02.jpg")),
|
|
file_hash=str(page_number) * 64,
|
|
file_size_bytes=1,
|
|
)
|
|
session.add(source)
|
|
await session.flush()
|
|
session.add(JobSource(job_id=job.id, source_id=source.id))
|
|
await session.commit()
|
|
loaded = await services.jobs.read_job(job_id=job.id, session=session)
|
|
|
|
second_started = asyncio.Event()
|
|
release_second = asyncio.Event()
|
|
call_count = 0
|
|
|
|
async def _transcribe(image_path, **kwargs):
|
|
nonlocal call_count
|
|
_ = (image_path, kwargs)
|
|
call_count += 1
|
|
if call_count == 2:
|
|
second_started.set()
|
|
await release_second.wait()
|
|
return TranscriptionResult(text=f"page {call_count}", provider="fixture", model="model")
|
|
|
|
monkeypatch.setattr("transcription.services.workflows.transcribe_document_image", _transcribe)
|
|
task = asyncio.create_task(process_queued_job(job=loaded, services=services))
|
|
await asyncio.wait_for(second_started.wait(), timeout=2)
|
|
|
|
attempts = await services.evidence.list_execution_attempts(job_id=job.id)
|
|
assert len(attempts) == 1
|
|
assert attempts[0].raw_transcription == "page 1"
|
|
|
|
release_second.set()
|
|
result = await task
|
|
assert result is not None
|
|
assert result.status == JobStatus.TRANSCRIBED
|