feat: run stale-job recovery periodically

Co-authored-by: Copilot App <[email protected]>

src/transcription/app.py

@@ -71,11 +71,11 @@ async def _lifespan(app: FastAPI):
 async def _recover_stale_processing_jobs(app: FastAPI) -> None:
     """Re-queue stale processing jobs at startup.
 
-    Any job left in PROCESSING longer than the configured provider timeout is
-    assumed orphaned and moved back to QUEUED before the worker starts.
+    Any job left in PROCESSING longer than the stale-job threshold is assumed
+    orphaned and moved back to QUEUED before the worker starts.
     """
     settings = app.state.settings
-    stale_before = datetime.now(UTC) - timedelta(seconds=settings.worker_provider_timeout_seconds)
+    stale_before = datetime.now(UTC) - timedelta(seconds=settings.worker_stale_job_seconds)
     recovered = await app.state.services.jobs.requeue_stale_processing_jobs(stale_before=stale_before)
     if recovered > 0:
         logger.warning("Recovered %s stale processing job(s) at startup", recovered)

src/transcription/config.py

@@ -114,6 +114,7 @@ class Settings(BaseSettings):
     # Bounded only from below. Vision transcription of a dense page routinely runs
     # well past twenty seconds, so an upper cap here would silently fail real work.
     worker_provider_timeout_seconds: float = Field(default=30.0, gt=0.0)
+    worker_stale_job_seconds: float = Field(default=30.0, gt=0.0)
     worker_min_transcription_chars: int = Field(default=0, ge=0)
     worker_min_transcription_lines: int = Field(default=0, ge=0)
     worker_fail_on_finish_reason_length: bool = False

src/transcription/worker.py

@@ -9,6 +9,9 @@ from contextlib import asynccontextmanager
 from contextlib import contextmanager
 from contextlib import suppress
 from dataclasses import dataclass
+from datetime import UTC
+from datetime import datetime
+from datetime import timedelta
 from typing import Literal
 from typing import Protocol
 from typing import runtime_checkable
@@ -215,6 +218,14 @@ async def run_worker_loop(
                     await asyncio.wait_for(wake_event.wait(), timeout=poll_interval_seconds)
                     wake_event.clear()
 
+            if session_factory is not None:
+                with handle_worker_exceptions(operation="worker.requeue_stale_processing_jobs"):
+                    stale_seconds = services.jobs.settings.worker_stale_job_seconds
+                    stale_before = datetime.now(UTC) - timedelta(seconds=stale_seconds)
+                    recovered = await services.jobs.requeue_stale_processing_jobs(stale_before=stale_before)
+                    if recovered > 0:
+                        logger.warning("Recovered %s stale processing job(s) in worker loop", recovered)
+
             processed_any = False
             while True:
                 with handle_worker_exceptions(operation="worker.process_next_queued_job"):