fix: gate retries by error category with backoff

Co-authored-by: Copilot App <[email protected]>

src/transcription/config.py

@@ -115,6 +115,7 @@ class Settings(BaseSettings):
     # well past twenty seconds, so an upper cap here would silently fail real work.
     worker_provider_timeout_seconds: float = Field(default=30.0, gt=0.0)
     worker_stale_job_seconds: float = Field(default=30.0, gt=0.0)
+    worker_retry_backoff_seconds: float = Field(default=1.0, ge=0.0)
     worker_min_transcription_chars: int = Field(default=0, ge=0)
     worker_min_transcription_lines: int = Field(default=0, ge=0)
     worker_fail_on_finish_reason_length: bool = False

src/transcription/services/evidence.py

@@ -43,6 +43,26 @@ class LatestExecutionAttempt:
 class EvidenceService(ServiceBase):
     """Read, project, and export execution attempt evidence."""
 
+    async def read_latest_job_error_category(
+        self,
+        *,
+        job_id: UUID,
+        session: AsyncSession | None = None,
+    ) -> str | None:
+        """Read the latest persisted execution-attempt error category for a job."""
+        async with self._session_scope(session) as _session:
+            query = (
+                select(ExecutionAttempt.error_category)
+                .where(ExecutionAttempt.job_id == job_id)
+                .where(col(ExecutionAttempt.error_category).is_not(None))
+                .order_by(
+                    col(ExecutionAttempt.created_at).desc(),
+                    col(ExecutionAttempt.id).desc(),
+                )
+                .limit(1)
+            )
+            return (await _session.exec(query)).first()
+
     async def read_latest_execution_attempt(
         self,
         *,

src/transcription/services/workflows.py

@@ -40,6 +40,12 @@ from .sources import transcribe_document_image
 
 logger = logging.getLogger(__name__)
 
+_RETRIABLE_FAILED_JOB_ERROR_CATEGORIES = {
+    ErrorCategory.EXTERNAL_PROVIDER.value,
+    ErrorCategory.EXTERNAL_TIMEOUT.value,
+    ErrorCategory.INFRA_TRANSIENT.value,
+}
+
 
 async def create_document_with_people(
     *,
@@ -182,16 +188,33 @@ async def advance_job(
             # Recover mid-flight jobs by continuing the queued processing path.
             return await process_queued_job(job=job, services=services, settings=settings, session=session)
         case JobStatus.FAILED:
-            if job.retry_count < settings.worker_max_retries:
+            latest_error_category = await services.evidence.read_latest_job_error_category(
+                job_id=job.id,
+                session=session,
+            )
+            can_retry = (
+                job.retry_count < settings.worker_max_retries
+                and latest_error_category in _RETRIABLE_FAILED_JOB_ERROR_CATEGORIES
+            )
+            if can_retry:
+                if settings.worker_retry_backoff_seconds > 0:
+                    await asyncio.sleep(settings.worker_retry_backoff_seconds)
                 return await services.jobs.update_job_state(
                     job_id=job.id,
                     status=JobStatus.QUEUED,
                     retry_count_increment=1,
                     session=session,
                 )
+
+            if job.retry_count < settings.worker_max_retries:
+                logger.warning(
+                    "Job %s failed with non-retriable category %s; skipping retry.",
+                    job.id,
+                    latest_error_category or "unknown",
+                )
             else:
                 logger.error("Job %s has failed and reached max retries.", job.id)
-                return
+            return
         case _:
             return