From a975ca299ae4706bc7c79cd34229d4025fea2d34 Mon Sep 17 00:00:00 2001 From: Jim Lancaster <40281233+zoltan57@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:58:52 -0500 Subject: [PATCH] Trouble shooting PDF transcriptions --- .gitignore | 6 +- README.md | 6 ++ src/transcription/config.py | 3 + src/transcription/providers/base.py | 4 ++ src/transcription/providers/openrouter.py | 47 ++++++++++++++- src/transcription/services/workflows.py | 72 +++++++++++++++++++++++ 6 files changed, 133 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 985436e..fcb4604 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,5 @@ wheels/ # SQLite database *.db -upload/ -*.jpg -*.jpeg -*.png +# Document images +uploads/* diff --git a/README.md b/README.md index e152d6c..f9af9f4 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,12 @@ Optional settings (defaults shown): DATABASE_URL=sqlite:///./transcription.db UPLOAD_DIR=./uploads PROMPT_DIR=./prompts +WORKER_MAX_RETRIES=0 +WORKER_RETRY_BACKOFF_SECONDS=0 +WORKER_PROVIDER_TIMEOUT_SECONDS=20 +WORKER_MIN_TRANSCRIPTION_CHARS=0 +WORKER_MIN_TRANSCRIPTION_LINES=0 +WORKER_FAIL_ON_FINISH_REASON_LENGTH=false ``` ### 3) Run the app diff --git a/src/transcription/config.py b/src/transcription/config.py index 7c20a0e..5657ed4 100644 --- a/src/transcription/config.py +++ b/src/transcription/config.py @@ -52,6 +52,9 @@ class Settings(BaseSettings): worker_max_retries: int = 0 worker_retry_backoff_seconds: float = 0.0 worker_provider_timeout_seconds: float = Field(default=20.0, gt=0.0, le=20.0) + worker_min_transcription_chars: int = Field(default=0, ge=0) + worker_min_transcription_lines: int = Field(default=0, ge=0) + worker_fail_on_finish_reason_length: bool = False @property def should_bootstrap_schema(self) -> bool: diff --git a/src/transcription/providers/base.py b/src/transcription/providers/base.py index 9056f45..6f68d4b 100644 --- a/src/transcription/providers/base.py +++ b/src/transcription/providers/base.py @@ -24,6 +24,10 @@ class TranscriptionResult: provider: str prompt_name: str model: str + finish_reason: str | None = None + usage_input_tokens: int | None = None + usage_output_tokens: int | None = None + usage_total_tokens: int | None = None class TranscriptionProvider(Protocol): diff --git a/src/transcription/providers/openrouter.py b/src/transcription/providers/openrouter.py index 4e6b9e2..86292c2 100644 --- a/src/transcription/providers/openrouter.py +++ b/src/transcription/providers/openrouter.py @@ -64,8 +64,19 @@ class OpenRouterTranscriptionProvider: text = self._extract_text(response) model = self._get_optional_attr(response, "model") or self.model + finish_reason = self._extract_finish_reason(response) + usage_input_tokens, usage_output_tokens, usage_total_tokens = self._extract_usage(response) logger.info("OpenRouter transcription completed using model=%s", model) - return TranscriptionResult(text=text, provider="openrouter", prompt_name="", model=model) + return TranscriptionResult( + text=text, + provider="openrouter", + prompt_name="", + model=model, + finish_reason=finish_reason, + usage_input_tokens=usage_input_tokens, + usage_output_tokens=usage_output_tokens, + usage_total_tokens=usage_total_tokens, + ) def _build_request(self, *, prompt_text: str, image_bytes: bytes, mime_type: str) -> OpenRouterRequest: image_b64 = base64.b64encode(image_bytes).decode("ascii") @@ -104,6 +115,34 @@ class OpenRouterTranscriptionProvider: raise ProviderResponseError("OpenRouter response contained no transcription text") return text + def _extract_finish_reason(self, response: Any) -> str | None: + choices = self._get_optional_attr(response, "choices") + if not choices: + return None + first_choice = choices[0] + finish_reason = self._get_optional_attr(first_choice, "finish_reason") + if isinstance(finish_reason, str) and finish_reason.strip(): + return finish_reason.strip() + return None + + def _extract_usage(self, response: Any) -> tuple[int | None, int | None, int | None]: + usage = self._get_optional_attr(response, "usage") + if usage is None: + return None, None, None + + input_tokens = self._as_int(self._get_optional_attr(usage, "prompt_tokens")) + output_tokens = self._as_int(self._get_optional_attr(usage, "completion_tokens")) + total_tokens = self._as_int(self._get_optional_attr(usage, "total_tokens")) + + if input_tokens is None: + input_tokens = self._as_int(self._get_optional_attr(usage, "input_tokens")) + if output_tokens is None: + output_tokens = self._as_int(self._get_optional_attr(usage, "output_tokens")) + if total_tokens is None: + total_tokens = self._as_int(self._get_optional_attr(usage, "total")) + + return input_tokens, output_tokens, total_tokens + def _normalize_content(self, content: Any) -> str: if isinstance(content, str): return content.strip() @@ -127,3 +166,9 @@ class OpenRouterTranscriptionProvider: if isinstance(obj, dict): return obj.get(key) return getattr(obj, key, None) + + @staticmethod + def _as_int(value: Any) -> int | None: + if isinstance(value, int): + return value + return None diff --git a/src/transcription/services/workflows.py b/src/transcription/services/workflows.py index 3479841..306917a 100644 --- a/src/transcription/services/workflows.py +++ b/src/transcription/services/workflows.py @@ -71,12 +71,35 @@ async def process_queued_job( source_job = await services.jobs.read_job(job_id=job.id, session=session) source = _resolve_primary_source(source_job) assert source is not None, f"Job {job.id} has no associated source record." + started_at = asyncio.get_running_loop().time() try: result = await asyncio.wait_for( transcribe_document_image(source.file_path), timeout=runtime_settings.worker_provider_timeout_seconds, ) + elapsed_seconds = asyncio.get_running_loop().time() - started_at + logger.info( + "Provider response diagnostics operation=worker.provider_response " + "job_id=%s document_id=%s source_id=%s provider=%s model=%s " + "finish_reason=%s usage_input_tokens=%s usage_output_tokens=%s usage_total_tokens=%s " + "latency_seconds=%.3f text_chars=%s text_lines=%s", + job.id, + job.document_id, + source.id, + result.provider, + result.model, + result.finish_reason or "unknown", + result.usage_input_tokens, + result.usage_output_tokens, + result.usage_total_tokens, + elapsed_seconds, + len(result.text), + _line_count(result.text), + ) + + _validate_transcription_quality(result=result, settings=runtime_settings) + job = await _finalize_transcribed(job=job, services=services, result=result, session=session) logger.info( "Job transcribed operation=worker.process_job job_id=%s document_id=%s source_id=%s provider=%s", @@ -271,3 +294,52 @@ def _resolve_primary_source(job: Job) -> Source | None: if not job.sources: return None return job.sources[0] + + +def _validate_transcription_quality(*, result: TranscriptionResult, settings: Settings) -> None: + text_chars = len(result.text) + text_lines = _line_count(result.text) + + if settings.worker_fail_on_finish_reason_length and (result.finish_reason or "").lower() == "length": + raise AppError( + "Provider output appears truncated (finish_reason=length)", + category=ErrorCategory.EXTERNAL_PROVIDER, + suggestion=( + "Retry the job. If this repeats, use a faster model, reduce input complexity, " + "or increase provider output budget." + ), + retriable=True, + ) + + if settings.worker_min_transcription_chars > 0 and text_chars < settings.worker_min_transcription_chars: + raise AppError( + ( + "Transcription output below configured minimum character threshold " + f"({text_chars} < {settings.worker_min_transcription_chars})" + ), + category=ErrorCategory.EXTERNAL_PROVIDER, + suggestion=( + "Retry the job. If this repeats, switch model or raise minimum thresholds based on document type." + ), + retriable=True, + ) + + if settings.worker_min_transcription_lines > 0 and text_lines < settings.worker_min_transcription_lines: + raise AppError( + ( + "Transcription output below configured minimum line threshold " + f"({text_lines} < {settings.worker_min_transcription_lines})" + ), + category=ErrorCategory.EXTERNAL_PROVIDER, + suggestion=( + "Retry the job. If this repeats, switch model or raise minimum thresholds based on document type." + ), + retriable=True, + ) + + +def _line_count(text: str) -> int: + stripped = text.strip() + if not stripped: + return 0 + return sum(1 for line in stripped.splitlines() if line.strip())