Jobs: big jobs stuck in queue. Added Cancel, Resubmit

2026-08-04 09:03:51 -05:00
parent 759d4c2434
commit 6c6589d8ff
11 changed files with 1126 additions and 124 deletions
@@ -1,25 +1,49 @@
 """Integration tests for end-to-end upload and worker pipeline behavior."""

 from pathlib import Path
+from uuid import uuid4

 import pytest

 from transcription.config import Settings
+from transcription.db.models import Document
 from transcription.db.models import Job
+from transcription.db.models import JobSourceStatus
 from transcription.db.models import JobStatus
 from transcription.providers.base import TranscriptionResult
 from transcription.services import ServiceBundle
+from transcription.services.store import create_job_for_document
 from transcription.services.store import create_upload_job
 from transcription.services.workflows import advance_job


+def _build_services(default_session_factory) -> ServiceBundle:
+    services = ServiceBundle()
+    object.__setattr__(
+        services,
+        "documents",
+        services.documents.__class__(session_factory=default_session_factory),
+    )
+    object.__setattr__(
+        services,
+        "jobs",
+        services.jobs.__class__(session_factory=default_session_factory),
+    )
+    object.__setattr__(
+        services,
+        "transcriptions",
+        services.transcriptions.__class__(session_factory=default_session_factory),
+    )
+    return services
+
+
@pytest.mark.integration
 class TestPipelineSuccessFlow:
    """Verify end-to-end success lifecycle behavior."""

    @pytest.mark.asyncio
    async def test_upload_then_worker_persists_transcribed_terminal_state(
-        self, async_session, tmp_path: Path, monkeypatch
+        self, async_session, default_session_factory, tmp_path: Path, monkeypatch
    ):
        """Upload followed by worker processing persists job transcription and transcribed status."""
        settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
@@ -59,12 +83,12 @@ class TestPipelineSuccessFlow:
            _fake_transcribe_document_image,
        )

-        services = ServiceBundle()
-        queued_job = await services.jobs.read_next_queued_job(session=async_session)
+        services = _build_services(default_session_factory)
+        queued_job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
        processed = queued_job is not None
        if queued_job is not None:
            await advance_job(job=queued_job, services=services, session=async_session)
-        job = await async_session.get(Job, upload_result.job_id)
+        job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)

        assert processed is True
        assert job is not None
@@ -72,13 +96,212 @@ class TestPipelineSuccessFlow:
        assert any(job_source.raw_transcription == "Pipeline transcript" for job_source in job.job_sources)
        assert all(job_source.error_detail is None for job_source in job.job_sources)

+    @pytest.mark.asyncio
+    async def test_worker_transcribes_all_sources_for_multi_page_job(
+        self,
+        async_session,
+        default_session_factory,
+        tmp_path: Path,
+        monkeypatch,
+    ):
+        """Worker stores transcription output for every source linked to the queued job."""
+        settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
+        document = Document(id=uuid4(), name="multi-page-document")
+        async_session.add(document)
+        await async_session.commit()
+
+        create_result = await create_job_for_document(
+            document_id=document.id,
+            uploads=[
+                ("page-01.jpg", b"one"),
+                ("page-02.jpg", b"two"),
+                ("page-03.jpg", b"three"),
+            ],
+            session=async_session,
+            settings=settings,
+        )
+
+        async def _fake_transcribe_document_image(
+            image_path,
+            *,
+            prompt_name="transcribe_document.md",
+            settings=None,
+            provider=None,
+        ) -> TranscriptionResult:
+            page_name = Path(image_path).name
+            _ = (prompt_name, settings, provider)
+            return TranscriptionResult(
+                text=f"Transcript for {page_name}",
+                provider="openrouter",
+                model="test-model",
+                prompt_name="transcribe_document.md",
+            )
+
+        monkeypatch.setattr(
+            "transcription.services.workflows.transcribe_document_image",
+            _fake_transcribe_document_image,
+        )
+
+        services = _build_services(default_session_factory)
+        queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
+        assert queued_job is not None
+
+        await advance_job(job=queued_job, services=services, session=async_session)
+        job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
+
+        assert job.status == JobStatus.TRANSCRIBED
+        assert len(job.job_sources) == 3
+        assert all(job_source.status == JobSourceStatus.TRANSCRIBED for job_source in job.job_sources)
+        assert all(job_source.raw_transcription for job_source in job.job_sources)
+        assert all(job_source.source is not None and job_source.source.raw_transcription for job_source in job.job_sources)
+
+    @pytest.mark.asyncio
+    async def test_worker_marks_partial_success_when_some_sources_fail(
+        self,
+        async_session,
+        default_session_factory,
+        tmp_path: Path,
+        monkeypatch,
+    ):
+        """Mixed page outcomes produce PARTIAL_SUCCESS and preserve per-source status."""
+        settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
+        document = Document(id=uuid4(), name="partial-page-document")
+        async_session.add(document)
+        await async_session.commit()
+
+        create_result = await create_job_for_document(
+            document_id=document.id,
+            uploads=[
+                ("page-01.jpg", b"one"),
+                ("page-02.jpg", b"two"),
+            ],
+            session=async_session,
+            settings=settings,
+        )
+
+        call_count = 0
+
+        async def _fake_transcribe_document_image(
+            image_path,
+            *,
+            prompt_name="transcribe_document.md",
+            settings=None,
+            provider=None,
+        ) -> TranscriptionResult:
+            nonlocal call_count
+            call_count += 1
+            _ = (prompt_name, settings, provider)
+            if call_count == 2:
+                raise RuntimeError("simulated page failure")
+            return TranscriptionResult(
+                text="Transcript for first page",
+                provider="openrouter",
+                model="test-model",
+                prompt_name="transcribe_document.md",
+            )
+
+        monkeypatch.setattr(
+            "transcription.services.workflows.transcribe_document_image",
+            _fake_transcribe_document_image,
+        )
+
+        services = _build_services(default_session_factory)
+        queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
+        assert queued_job is not None
+
+        await advance_job(job=queued_job, services=services, session=async_session)
+        job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
+
+        assert job.status == JobStatus.PARTIAL_SUCCESS
+        assert len(job.job_sources) == 2
+        statuses = {job_source.status for job_source in job.job_sources}
+        assert statuses == {JobSourceStatus.TRANSCRIBED, JobSourceStatus.FAILED}
+        assert any(job_source.error_detail is not None for job_source in job.job_sources)
+
+    @pytest.mark.asyncio
+    async def test_worker_skips_already_transcribed_sources_on_resubmit(
+        self,
+        async_session,
+        default_session_factory,
+        tmp_path: Path,
+        monkeypatch,
+    ):
+        """Queued jobs only process non-transcribed JobSource records."""
+        settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
+        document = Document(id=uuid4(), name="resubmit-filter-document")
+        async_session.add(document)
+        await async_session.commit()
+
+        create_result = await create_job_for_document(
+            document_id=document.id,
+            uploads=[
+                ("page-01.jpg", b"one"),
+                ("page-02.jpg", b"two"),
+            ],
+            session=async_session,
+            settings=settings,
+        )
+
+        services = _build_services(default_session_factory)
+        job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
+        page_one = next(js for js in job.job_sources if js.source is not None and js.source.page_number == 1)
+        page_two = next(js for js in job.job_sources if js.source is not None and js.source.page_number == 2)
+
+        page_one.status = JobSourceStatus.TRANSCRIBED
+        page_one.raw_transcription = "existing transcript"
+        page_two.status = JobSourceStatus.PENDING
+        page_two.raw_transcription = None
+        await services.transcriptions.update_job_source(job_source=page_one, session=async_session)
+        await services.transcriptions.update_job_source(job_source=page_two, session=async_session)
+        await services.jobs.update_job_state(job_id=job.id, status=JobStatus.QUEUED, session=async_session)
+        await async_session.commit()
+
+        call_count = 0
+
+        async def _fake_transcribe_document_image(
+            image_path,
+            *,
+            prompt_name="transcribe_document.md",
+            settings=None,
+            provider=None,
+        ) -> TranscriptionResult:
+            nonlocal call_count
+            _ = (image_path, prompt_name, settings, provider)
+            call_count += 1
+            return TranscriptionResult(
+                text="new transcript",
+                provider="openrouter",
+                model="test-model",
+                prompt_name="transcribe_document.md",
+            )
+
+        monkeypatch.setattr(
+            "transcription.services.workflows.transcribe_document_image",
+            _fake_transcribe_document_image,
+        )
+
+        queued_job = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
+        assert queued_job is not None
+        await advance_job(job=queued_job, services=services, session=async_session)
+
+        refreshed = await services.jobs.read_job(job_id=create_result.job_id, session=async_session)
+        assert call_count == 1
+        statuses = {js.status for js in refreshed.job_sources}
+        assert statuses == {JobSourceStatus.TRANSCRIBED}
+

@pytest.mark.integration
 class TestPipelineFailureFlow:
    """Verify end-to-end failure lifecycle behavior."""

    @pytest.mark.asyncio
-    async def test_upload_then_worker_persists_failed_terminal_state(self, async_session, tmp_path: Path, monkeypatch):
+    async def test_upload_then_worker_persists_failed_terminal_state(
+        self,
+        async_session,
+        default_session_factory,
+        tmp_path: Path,
+        monkeypatch,
+    ):
        """Upload followed by worker processing persists error detail and failed status on the job."""
        settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
        upload_result = await create_upload_job(
@@ -103,12 +326,12 @@ class TestPipelineFailureFlow:
            _fake_transcribe_document_image,
        )

-        services = ServiceBundle()
-        queued_job = await services.jobs.read_next_queued_job(session=async_session)
+        services = _build_services(default_session_factory)
+        queued_job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)
        processed = queued_job is not None
        if queued_job is not None:
            await advance_job(job=queued_job, services=services, session=async_session)
-        job = await async_session.get(Job, upload_result.job_id)
+        job = await services.jobs.read_job(job_id=upload_result.job_id, session=async_session)

        assert processed is True
        assert job is not None
@@ -13,6 +13,8 @@ from transcription.db.models import JobStatus
 from transcription.db.models import Source
 from transcription.services.documents import DocumentService
 from transcription.services.jobs import JobDeleteBlockedError
+from transcription.services.jobs import JobCancelBlockedError
+from transcription.services.jobs import JobResubmitBlockedError
 from transcription.services.jobs import JobService


@@ -224,3 +226,157 @@ class TestJobService:

        with pytest.raises(ValueError):
            await job_service.read_job(job_id=job.id)
+
+    @pytest.mark.asyncio
+    async def test_cancel_job_marks_non_transcribed_sources_failed(
+        self,
+        job_service: JobService,
+        document_service: DocumentService,
+    ):
+        document = Document(id=uuid4(), name="cancel-job-doc")
+        await document_service.create_document(document=document)
+
+        job = Job(document_id=document.id, status=JobStatus.QUEUED)
+        await job_service.create_job(job=job)
+
+        async with job_service._session_scope() as session:
+            source_one = Source(
+                document_id=document.id,
+                page_number=1,
+                upload_name="cancel-1.jpg",
+                filename="stored-cancel-1.jpg",
+                file_path="/uploads/stored-cancel-1.jpg",
+            )
+            source_two = Source(
+                document_id=document.id,
+                page_number=2,
+                upload_name="cancel-2.jpg",
+                filename="stored-cancel-2.jpg",
+                file_path="/uploads/stored-cancel-2.jpg",
+            )
+            session.add(source_one)
+            session.add(source_two)
+            await session.flush()
+
+            session.add(
+                JobSource(
+                    job_id=job.id,
+                    source_id=source_one.id,
+                    status=JobSourceStatus.TRANSCRIBED,
+                    raw_transcription="done",
+                )
+            )
+            session.add(
+                JobSource(
+                    job_id=job.id,
+                    source_id=source_two.id,
+                    status=JobSourceStatus.PENDING,
+                )
+            )
+            await session.commit()
+
+        cancelled = await job_service.cancel_job(job_id=job.id)
+        assert cancelled.status == JobStatus.FAILED
+
+        refreshed = await job_service.read_job(job_id=job.id)
+        statuses = {item.status for item in refreshed.job_sources}
+        assert JobSourceStatus.TRANSCRIBED in statuses
+        assert JobSourceStatus.FAILED in statuses
+        pending_entry = next(item for item in refreshed.job_sources if item.status == JobSourceStatus.FAILED)
+        assert pending_entry.error_detail == "Cancelled by user"
+
+    @pytest.mark.asyncio
+    async def test_resubmit_non_transcribed_sources_resets_only_non_transcribed(
+        self,
+        job_service: JobService,
+        document_service: DocumentService,
+    ):
+        document = Document(id=uuid4(), name="resubmit-job-doc")
+        await document_service.create_document(document=document)
+
+        job = Job(document_id=document.id, status=JobStatus.FAILED)
+        await job_service.create_job(job=job)
+
+        async with job_service._session_scope() as session:
+            source_one = Source(
+                document_id=document.id,
+                page_number=1,
+                upload_name="resubmit-1.jpg",
+                filename="stored-resubmit-1.jpg",
+                file_path="/uploads/stored-resubmit-1.jpg",
+                raw_transcription="existing text",
+            )
+            source_two = Source(
+                document_id=document.id,
+                page_number=2,
+                upload_name="resubmit-2.jpg",
+                filename="stored-resubmit-2.jpg",
+                file_path="/uploads/stored-resubmit-2.jpg",
+                raw_transcription="done text",
+            )
+            session.add(source_one)
+            session.add(source_two)
+            await session.flush()
+
+            session.add(
+                JobSource(
+                    job_id=job.id,
+                    source_id=source_one.id,
+                    status=JobSourceStatus.FAILED,
+                    raw_transcription=None,
+                    error_detail="prior error",
+                )
+            )
+            session.add(
+                JobSource(
+                    job_id=job.id,
+                    source_id=source_two.id,
+                    status=JobSourceStatus.TRANSCRIBED,
+                    raw_transcription="done text",
+                )
+            )
+            await session.commit()
+
+        count = await job_service.resubmit_non_transcribed_sources(job_id=job.id)
+        assert count == 1
+
+        refreshed = await job_service.read_job(job_id=job.id)
+        assert refreshed.status == JobStatus.QUEUED
+
+        failed_entry = next(item for item in refreshed.job_sources if item.source is not None and item.source.page_number == 1)
+        transcribed_entry = next(item for item in refreshed.job_sources if item.source is not None and item.source.page_number == 2)
+        assert failed_entry.status == JobSourceStatus.PENDING
+        assert failed_entry.error_detail is None
+        assert failed_entry.source is not None
+        assert failed_entry.source.raw_transcription is None
+        assert transcribed_entry.status == JobSourceStatus.TRANSCRIBED
+
+    @pytest.mark.asyncio
+    async def test_resubmit_non_transcribed_sources_blocks_when_processing(
+        self,
+        job_service: JobService,
+        document_service: DocumentService,
+    ):
+        document = Document(id=uuid4(), name="resubmit-blocked-doc")
+        await document_service.create_document(document=document)
+
+        job = Job(document_id=document.id, status=JobStatus.PROCESSING)
+        await job_service.create_job(job=job)
+
+        with pytest.raises(JobResubmitBlockedError):
+            await job_service.resubmit_non_transcribed_sources(job_id=job.id)
+
+    @pytest.mark.asyncio
+    async def test_cancel_job_blocks_transcribed_terminal_jobs(
+        self,
+        job_service: JobService,
+        document_service: DocumentService,
+    ):
+        document = Document(id=uuid4(), name="cancel-blocked-doc")
+        await document_service.create_document(document=document)
+
+        job = Job(document_id=document.id, status=JobStatus.TRANSCRIBED)
+        await job_service.create_job(job=job)
+
+        with pytest.raises(JobCancelBlockedError):
+            await job_service.cancel_job(job_id=job.id)
@@ -154,3 +154,54 @@ class TestTranscriptionServiceRevisionUpsert:

        with pytest.raises(SourceDeleteBlockedError):
            await transcriptions.delete_source_from_job_context(job_id=job_one.id, source_id=source.id)
+
+    @pytest.mark.asyncio
+    async def test_delete_unlinked_source_succeeds(self, default_session_factory):
+        documents = DocumentService(session_factory=default_session_factory)
+        transcriptions = TranscriptionService(session_factory=default_session_factory)
+
+        document = Document(id=uuid4(), name="delete-unlinked-source")
+        await documents.create_document(document=document)
+
+        source = Source(
+            document_id=document.id,
+            page_number=1,
+            upload_name="orphan.jpg",
+            filename="orphan.jpg",
+            file_path="uploads/orphan.jpg",
+        )
+        await transcriptions.create_source(source=source)
+
+        await transcriptions.delete_unlinked_source(source_id=source.id)
+
+        with pytest.raises(TranscriptionNotFoundError):
+            await transcriptions.read_source(source.id)
+
+    @pytest.mark.asyncio
+    async def test_delete_unlinked_source_blocks_when_linked(self, default_session_factory):
+        documents = DocumentService(session_factory=default_session_factory)
+        jobs = JobService(session_factory=default_session_factory)
+        transcriptions = TranscriptionService(session_factory=default_session_factory)
+
+        document = Document(id=uuid4(), name="delete-unlinked-blocked")
+        await documents.create_document(document=document)
+
+        job = Job(document_id=document.id, status=JobStatus.QUEUED)
+        await jobs.create_job(job=job)
+
+        source = Source(
+            document_id=document.id,
+            page_number=1,
+            upload_name="linked.jpg",
+            filename="linked.jpg",
+            file_path="uploads/linked.jpg",
+        )
+        async with transcriptions._session_scope() as session:
+            session.add(source)
+            await session.flush()
+            session.add(JobSource(job_id=job.id, source_id=source.id, status=JobSourceStatus.PENDING))
+            await session.commit()
+            await session.refresh(source)
+
+        with pytest.raises(SourceDeleteBlockedError):
+            await transcriptions.delete_unlinked_source(source_id=source.id)
@@ -86,7 +86,7 @@ class TestPageRendering:
        assert "document links" in response.text.lower()
        assert "Sources" in response.text
        assert "Jobs" in response.text
-        assert "Delete job" not in response.text
+        assert "Delete Job" in response.text

    def test_job_detail_page_rejects_invalid_id(self, app_client):
        """GET /ui/jobs/{job_id} shows validation feedback for malformed IDs."""
@@ -105,19 +105,41 @@ class TestPageRendering:
        assert response.status_code == 200
        assert "Job not found" in response.text

-    def test_job_detail_page_hides_delete_action(self, app_client, seed_job):
-        """GET /ui/jobs/{job_id} does not expose job deletion controls in this revision."""
+    def test_job_detail_page_shows_cancel_and_resubmit_when_queued(self, app_client, seed_job):
+        """GET /ui/jobs/{job_id} exposes cancel/resubmit controls for queued jobs."""
        _, client = app_client
        job_id = seed_job(
            filename="no-revision.pdf",
-            status=JobStatus.TRANSCRIBED,
-            transcription_text="original text",
+            status=JobStatus.QUEUED,
+            transcription_text=None,
        )

        response = client.get(f"/ui/jobs/{job_id}")

        assert response.status_code == 200
-        assert "Delete job" not in response.text
+        assert "Cancel" in response.text
+        assert "Resubmit" in response.text
+        assert "Delete Job" in response.text
+
+    def test_job_cancel_page_renders_confirmation(self, app_client, seed_job):
+        _, client = app_client
+        job_id = seed_job(filename="cancel-ready.pdf", status=JobStatus.PROCESSING)
+
+        response = client.get(f"/ui/jobs/{job_id}/cancel")
+
+        assert response.status_code == 200
+        assert "Cancel Processing Job" in response.text
+        assert "Cancel job" in response.text
+
+    def test_job_resubmit_page_renders_confirmation(self, app_client, seed_job):
+        _, client = app_client
+        job_id = seed_job(filename="resubmit-ready.pdf", status=JobStatus.FAILED, transcription_text=None)
+
+        response = client.get(f"/ui/jobs/{job_id}/resubmit")
+
+        assert response.status_code == 200
+        assert "Resubmit Job" in response.text
+        assert "Resubmit now" in response.text

    def test_job_delete_page_shows_confirmation_when_not_processing(self, app_client, seed_job):
        _, client = app_client
@@ -138,3 +138,52 @@ class TestSourcesPageRendering:
        assert "human revision text" in response.text
        assert "Page Number:" in response.text
        assert "Stored Filename:" in response.text
+        assert "Delete Source" in response.text
+
+    def test_source_delete_page_blocks_when_source_is_job_linked(self, app_client, seed_job):
+        _, client = app_client
+        job_id = seed_job(filename="linked-source.png", transcription_text="linked text")
+
+        async def _get_source_id() -> str:
+            async with session_scope() as session:
+                job = await session.get(Job, job_id)
+                assert job is not None
+                source = (
+                    await session.exec(select(Source).where(Source.document_id == job.document_id))
+                ).first()
+                assert source is not None
+                return str(source.id)
+
+        source_id = asyncio.run(_get_source_id())
+        response = client.get(f"/ui/sources/{source_id}/delete")
+
+        assert response.status_code == 200
+        assert "Delete Source Record" in response.text
+        assert "Delete is only available for unlinked sources." in response.text
+
+    def test_source_delete_page_allows_unlinked_source(self, app_client):
+        _, client = app_client
+
+        async def _seed_unlinked_source() -> str:
+            async with session_scope() as session:
+                document = Document(name="Unlinked Source Doc", document_type="memo")
+                session.add(document)
+                await session.flush()
+                source = Source(
+                    document_id=document.id,
+                    page_number=1,
+                    upload_name="orphan-source.png",
+                    filename="orphan-source.png",
+                    file_path="/tmp/orphan-source.png",
+                )
+                session.add(source)
+                await session.commit()
+                return str(source.id)
+
+        source_id = asyncio.run(_seed_unlinked_source())
+        response = client.get(f"/ui/sources/{source_id}/delete")
+
+        assert response.status_code == 200
+        assert "Delete Source Record" in response.text
+        assert "Delete source permanently" in response.text
+        assert "Delete is only available for unlinked sources." not in response.text