generated from john/python-template
V4.5 Complete - Enhanced trancription context, added option to restranscribe source under different models.
This commit is contained in:
@@ -12,6 +12,8 @@ OPENROUTER_API_KEY=your-api-key-goes-here
|
|||||||
# GEMINI_API_KEY - Required when `PROVIDER=google_genai`
|
# GEMINI_API_KEY - Required when `PROVIDER=google_genai`
|
||||||
# PROVIDER_MODEL= specify model. If left blank OpenRouter will supply default.
|
# PROVIDER_MODEL= specify model. If left blank OpenRouter will supply default.
|
||||||
PROVIDER_MODEL=google/gemini-2.5-flash
|
PROVIDER_MODEL=google/gemini-2.5-flash
|
||||||
|
# Optional JSON allowlist for model selection. The default above is always first.
|
||||||
|
# PROVIDER_MODELS=["google/gemini-2.5-flash","google/gemini-2.5-pro","anthropic/claude-sonnet-4"]
|
||||||
# OPENROUTER_HTTP_REFERER=https://example.com
|
# OPENROUTER_HTTP_REFERER=https://example.com
|
||||||
# OPENROUTER_APP_TITLE="Google: Gemini 2.5 Flash (openrouter)"
|
# OPENROUTER_APP_TITLE="Google: Gemini 2.5 Flash (openrouter)"
|
||||||
|
|
||||||
|
|||||||
+1
-2
@@ -57,5 +57,4 @@ Each page contract contains:
|
|||||||
|
|
||||||
## Current Baseline
|
## Current Baseline
|
||||||
|
|
||||||
These contracts describe the completed V4 through V4.4 behavior.
|
These contracts describe the completed V4 through V4.5 behavior.
|
||||||
V4.5 planned behavior remains defined by its frozen scope and implementation plan until implementation updates the applicable page contracts.
|
|
||||||
|
|||||||
@@ -36,6 +36,8 @@ Jobs manages transcription processing runs. A Job belongs to one Document, links
|
|||||||
- Files can be removed individually or cleared before submission.
|
- Files can be removed individually or cleared before submission.
|
||||||
- Helper text explains numeric filename prefixes for page ordering.
|
- Helper text explains numeric filename prefixes for page ordering.
|
||||||
- Submission creates the Job, Source records, and JobSource links, notifies the worker, and opens Job Detail.
|
- Submission creates the Job, Source records, and JobSource links, notifies the worker, and opens Job Detail.
|
||||||
|
- When opened with `source_id`, creation becomes a retranscription flow: Source and Document are locked, Provider is
|
||||||
|
read-only, Model is restricted to `PROVIDER_MODELS`, no upload is accepted, and one existing Source is linked.
|
||||||
|
|
||||||
## Detail and Lifecycle Behavior
|
## Detail and Lifecycle Behavior
|
||||||
|
|
||||||
@@ -61,7 +63,7 @@ Jobs manages transcription processing runs. A Job belongs to one Document, links
|
|||||||
- The page explains that resubmission queues failed linked Sources while preserving immutable prior attempt evidence.
|
- The page explains that resubmission queues failed linked Sources while preserving immutable prior attempt evidence.
|
||||||
- The service blocks submission while processing is active or when no failed Sources exist.
|
- The service blocks submission while processing is active or when no failed Sources exist.
|
||||||
- `JobSource` remains the latest compatibility projection, while every provider call appends an `ExecutionAttempt`.
|
- `JobSource` remains the latest compatibility projection, while every provider call appends an `ExecutionAttempt`.
|
||||||
- The latest successful `Source.raw_transcription` projection remains available while a retry is pending or fails.
|
- The selected `Source.raw_transcription` projection remains available while a retry is pending or fails.
|
||||||
- Success reports the number of resubmitted Sources and returns to Job Detail.
|
- Success reports the number of resubmitted Sources and returns to Job Detail.
|
||||||
|
|
||||||
## Delete Behavior
|
## Delete Behavior
|
||||||
|
|||||||
@@ -30,14 +30,20 @@ The list accepts optional `document_id` and `job_id` query parameters. Document
|
|||||||
|
|
||||||
- The heading shows page number, upload name, and Source ID.
|
- The heading shows page number, upload name, and Source ID.
|
||||||
- **Back to Sources** returns to the global list.
|
- **Back to Sources** returns to the global list.
|
||||||
|
- **Retranscribe Source** opens Create Processing Job with this Source and its Document locked.
|
||||||
- **Delete Source** opens the guarded delete route.
|
- **Delete Source** opens the guarded delete route.
|
||||||
- Previous and Next navigate only among Sources belonging to the same Document in page order; unavailable boundary actions are disabled.
|
- Previous and Next navigate only among Sources belonging to the same Document in page order; unavailable boundary actions are disabled.
|
||||||
- The media viewer resolves the stored Source path through the configured upload root.
|
- The media viewer resolves the stored Source path through the configured upload root.
|
||||||
- Transcription Text is read-only and prefers the latest JobSource transcription, then the Source projection.
|
- Transcription Text is read-only and displays the preferred machine projection, with a legacy latest-JobSource
|
||||||
|
fallback only when no Source projection exists.
|
||||||
- Editable Revision is seeded from an existing revision or the machine transcription.
|
- Editable Revision is seeded from an existing revision or the machine transcription.
|
||||||
- Source Metadata shows upload name, stored filename, page number, Document Name, Document ID, and stored path. Source ID appears in the page-header subtitle.
|
- Source Metadata shows upload name, stored filename, page number, Document Name, Document ID, and stored path. Source ID appears in the page-header subtitle.
|
||||||
- SourceJob Metadata shows latest status, Job ID, execution time, provider, model, prompt, and failure detail.
|
- SourceJob Metadata shows latest status, Job ID, execution time, provider, model, prompt, and failure detail.
|
||||||
- Revision Logistics shows revised state, last-revised time, and upload time.
|
- Revision Logistics shows revised state, last-revised time, and upload time.
|
||||||
|
- Candidate Machine Transcriptions remains compact until a candidate is expanded, then compares it with the preferred
|
||||||
|
machine result and requires confirmation before **Use this transcription**.
|
||||||
|
- Candidate promotion does not alter a human revision. Empty states distinguish no machine result from no candidates.
|
||||||
|
- An orientation-normalized artifact appears in evidence only when recognized metadata required a physical rotation.
|
||||||
|
|
||||||
## Provider Evidence
|
## Provider Evidence
|
||||||
|
|
||||||
@@ -47,6 +53,7 @@ The list accepts optional `document_id` and `job_id` query parameters. Document
|
|||||||
- Historical `raw_api_response` values are labeled as OpenRouter SDK response snapshots.
|
- Historical `raw_api_response` values are labeled as OpenRouter SDK response snapshots.
|
||||||
- Missing evidence has an explicit empty state.
|
- Missing evidence has an explicit empty state.
|
||||||
- Historical executions explicitly state that exact transport evidence was not captured.
|
- Historical executions explicitly state that exact transport evidence was not captured.
|
||||||
|
- Quality warning artifacts remain attached to their machine attempt and are not recomputed during page rendering.
|
||||||
- **Export Evidence** downloads a versioned package containing source identity, attempts, artifacts, relationships,
|
- **Export Evidence** downloads a versioned package containing source identity, attempts, artifacts, relationships,
|
||||||
schema versions, and integrity digests without source binaries, credentials, or machine-local source paths.
|
schema versions, and integrity digests without source binaries, credentials, or machine-local source paths.
|
||||||
|
|
||||||
@@ -71,6 +78,7 @@ The list accepts optional `document_id` and `job_id` query parameters. Document
|
|||||||
- List columns and alignments match this contract and omit Stored Filename.
|
- List columns and alignments match this contract and omit Stored Filename.
|
||||||
- Previous/next navigation never crosses Document boundaries.
|
- Previous/next navigation never crosses Document boundaries.
|
||||||
- Detail keeps machine output read-only and human revision separately editable.
|
- Detail keeps machine output read-only and human revision separately editable.
|
||||||
|
- Retranscription, candidate comparison, warnings, and explicit promotion preserve every prior attempt.
|
||||||
- Empty, failed, and missing-evidence states remain explicit.
|
- Empty, failed, and missing-evidence states remain explicit.
|
||||||
- JSON evidence is readable without being mislabeled as native transport evidence.
|
- JSON evidence is readable without being mislabeled as native transport evidence.
|
||||||
- Delete cannot remove a Source with processing-history links.
|
- Delete cannot remove a Source with processing-history links.
|
||||||
|
|||||||
@@ -137,11 +137,12 @@ Responsibilities:
|
|||||||
|
|
||||||
1. User uploads one or more images for a `Document`.
|
1. User uploads one or more images for a `Document`.
|
||||||
2. System stores files, hashes them, creates ordered `Source` rows, and creates a `Job`.
|
2. System stores files, hashes them, creates ordered `Source` rows, and creates a `Job`.
|
||||||
3. Worker claims the job, marks it `processing`, and executes page calls concurrently.
|
3. Worker claims the job, marks it `processing`, resolves metadata-directed orientation, and sends either the
|
||||||
|
immutable original or an exact normalized derivative to the provider.
|
||||||
4. Each provider call appends an `ExecutionAttempt` with its request manifest, transport evidence, SDK snapshot,
|
4. Each provider call appends an `ExecutionAttempt` with its request manifest, transport evidence, SDK snapshot,
|
||||||
normalized metadata, timing, and outcome.
|
normalized metadata, timing, and outcome.
|
||||||
5. The linked `JobSource` is updated as a compatibility projection, and a successful attempt updates the
|
5. The linked `JobSource` is updated as a compatibility projection. The first successful attempt establishes
|
||||||
`Source.raw_transcription` latest-success projection.
|
`Source.preferred_execution_attempt_id` and `Source.raw_transcription`; later successes remain candidates.
|
||||||
6. Aggregate status becomes `completed`, `partial_success`, or `failed`.
|
6. Aggregate status becomes `completed`, `partial_success`, or `failed`.
|
||||||
|
|
||||||
### 2. Document-Person Relationship Management
|
### 2. Document-Person Relationship Management
|
||||||
@@ -170,13 +171,18 @@ Responsibilities:
|
|||||||
## V4 Domain Rules
|
## V4 Domain Rules
|
||||||
|
|
||||||
- `JobSource.raw_transcription` preserves page output for its Job execution.
|
- `JobSource.raw_transcription` preserves page output for its Job execution.
|
||||||
- `Source.raw_transcription` is the latest-success machine-output projection for a page.
|
- `Source.raw_transcription` is the selected preferred-machine-output projection for a page.
|
||||||
|
- `Source.preferred_execution_attempt_id` identifies its exact immutable provenance; candidate promotion updates
|
||||||
|
both fields atomically.
|
||||||
- Human corrections occur only in `Source.revised_text`.
|
- Human corrections occur only in `Source.revised_text`.
|
||||||
- Prompt and parameter provenance is frozen on `Job` at submission time.
|
- Prompt and parameter provenance is frozen on `Job` at submission time.
|
||||||
- The SDK-serialized OpenRouter response snapshot is stored on `JobSource` for each successful page execution.
|
- The SDK-serialized OpenRouter response snapshot is stored on `JobSource` for each successful page execution.
|
||||||
- Every V4.2 provider call appends a distinct `ExecutionAttempt`; retries never rewrite earlier attempts.
|
- Every V4.2 provider call appends a distinct `ExecutionAttempt`; retries never rewrite earlier attempts.
|
||||||
- Exact response bytes identify the OpenRouter HTTP boundary and are not labeled as native upstream-provider JSON.
|
- Exact response bytes identify the OpenRouter HTTP boundary and are not labeled as native upstream-provider JSON.
|
||||||
- Generic `ProcessingArtifact` records use versioned schemas, digests, and one inline or external content location.
|
- Generic `ProcessingArtifact` records use versioned schemas, digests, and one inline or external content location.
|
||||||
|
- Orientation-normalized model inputs and deterministic quality warnings are versioned `ProcessingArtifact` evidence
|
||||||
|
attached to the consuming `ExecutionAttempt`.
|
||||||
|
- A `retranscription` Job contains one locked existing Source and freezes one configured allowlisted model.
|
||||||
- `DocumentPerson` links are unique for `(document_id, person_id)` and require one `role_id`.
|
- `DocumentPerson` links are unique for `(document_id, person_id)` and require one `role_id`.
|
||||||
- Relationship mutations are deterministic, set-based, and atomic with Document writes.
|
- Relationship mutations are deterministic, set-based, and atomic with Document writes.
|
||||||
- `DocumentType.id` and `PersonRole.id` are canonical relationship identities; unique labels may evolve.
|
- `DocumentType.id` and `PersonRole.id` are canonical relationship identities; unique labels may evolve.
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ This document defines the baseline requirements for the document transcription s
|
|||||||
| REQ-6 | Functional | Support an extensible role taxonomy for document-person relationships. | inspection |
|
| REQ-6 | Functional | Support an extensible role taxonomy for document-person relationships. | inspection |
|
||||||
| REQ-7 | Policy Constraint | Enforce deterministic relationship-role writes with uniqueness on `(document_id, person_id)` and explicit conflict responses for duplicate Person links. | test |
|
| REQ-7 | Policy Constraint | Enforce deterministic relationship-role writes with uniqueness on `(document_id, person_id)` and explicit conflict responses for duplicate Person links. | test |
|
||||||
| REQ-8 | Functional | Use set-based synchronization for document-person mutations so updates add and remove only the intended links. | test |
|
| REQ-8 | Functional | Use set-based synchronization for document-person mutations so updates add and remove only the intended links. | test |
|
||||||
| REQ-9 | Functional | Maintain immutable machine output on `Source.raw_transcription` while permitting inline human edits on `Source.revised_text`. | test |
|
| REQ-9 | Functional | Maintain selected machine output and exact attempt provenance on `Source` while permitting independent human edits on `Source.revised_text`. | test |
|
||||||
| REQ-10 | Functional | Support a UUID-identified `DocumentType` taxonomy with unique user-facing labels and active/inactive lifecycle control. | test |
|
| REQ-10 | Functional | Support a UUID-identified `DocumentType` taxonomy with unique user-facing labels and active/inactive lifecycle control. | test |
|
||||||
| REQ-11 | Data Constraint | Store `Document` type as a controlled reference to `DocumentType`. | test |
|
| REQ-11 | Data Constraint | Store `Document` type as a controlled reference to `DocumentType`. | test |
|
||||||
| REQ-12 | Interface | Render multi-page transcriptions sequentially by `page_number` with document, people, and document-type metadata. | demonstration |
|
| REQ-12 | Interface | Render multi-page transcriptions sequentially by `page_number` with document, people, and document-type metadata. | demonstration |
|
||||||
@@ -31,6 +31,9 @@ This document defines the baseline requirements for the document transcription s
|
|||||||
| REQ-22 | Interface | Provide safe browser-native Facsimile and Text-only print views from persisted Document Detail. | demonstration |
|
| REQ-22 | Interface | Provide safe browser-native Facsimile and Text-only print views from persisted Document Detail. | demonstration |
|
||||||
| REQ-23 | Security | Escape stored print text and serve Source images through record-validated application routes without disclosing local paths. | test |
|
| REQ-23 | Security | Escape stored print text and serve Source images through record-validated application routes without disclosing local paths. | test |
|
||||||
| REQ-24 | Functional | Print current human-preferred Source text, semantic Author metadata, deterministic Source order, and oldest-to-newest Job metadata. | test |
|
| REQ-24 | Functional | Print current human-preferred Source text, semantic Author metadata, deterministic Source order, and oldest-to-newest Job metadata. | test |
|
||||||
|
| REQ-25 | Quality | Physically apply recognized raster orientation metadata to provider-input derivatives without changing original Source bytes. | test |
|
||||||
|
| REQ-26 | Quality | Persist deterministic, non-mutating output warnings without automatic paid retries. | test |
|
||||||
|
| REQ-27 | Functional | Create one-Source retranscription Jobs from a configured model allowlist and preserve later successes as candidates until explicit promotion. | test |
|
||||||
|
|
||||||
## Clarifying Constraints
|
## Clarifying Constraints
|
||||||
|
|
||||||
|
|||||||
@@ -71,6 +71,7 @@ UUID id PK
|
|||||||
UUID document_id FK
|
UUID document_id FK
|
||||||
VARCHAR status
|
VARCHAR status
|
||||||
INTEGER retry_count
|
INTEGER retry_count
|
||||||
|
VARCHAR purpose
|
||||||
TEXT provider
|
TEXT provider
|
||||||
TEXT model
|
TEXT model
|
||||||
TEXT prompt_name
|
TEXT prompt_name
|
||||||
@@ -93,6 +94,7 @@ TEXT file_path
|
|||||||
TEXT file_hash
|
TEXT file_hash
|
||||||
BIGINT file_size_bytes
|
BIGINT file_size_bytes
|
||||||
TEXT raw_transcription
|
TEXT raw_transcription
|
||||||
|
UUID preferred_execution_attempt_id FK
|
||||||
TEXT revised_text
|
TEXT revised_text
|
||||||
TIMESTAMPTZ date_uploaded
|
TIMESTAMPTZ date_uploaded
|
||||||
TIMESTAMPTZ date_revised
|
TIMESTAMPTZ date_revised
|
||||||
@@ -177,7 +179,9 @@ EXECUTION_ATTEMPT ||--o{ PROCESSING_ARTIFACT : produces
|
|||||||
- `EXECUTION_ATTEMPT(job_id, source_id, attempt_number)` is unique; retries increment the persisted attempt number.
|
- `EXECUTION_ATTEMPT(job_id, source_id, attempt_number)` is unique; retries increment the persisted attempt number.
|
||||||
- Historical `JOB_SOURCE` rows without an `EXECUTION_ATTEMPT` remain SDK snapshots and are explicitly labeled as
|
- Historical `JOB_SOURCE` rows without an `EXECUTION_ATTEMPT` remain SDK snapshots and are explicitly labeled as
|
||||||
lacking transport evidence.
|
lacking transport evidence.
|
||||||
- `SOURCE.raw_transcription` caches the latest successful machine output for that page.
|
- `SOURCE.raw_transcription` caches the explicitly selected preferred machine output for that page.
|
||||||
|
- `SOURCE.preferred_execution_attempt_id` records exact successful-attempt provenance. Legacy projections may remain
|
||||||
|
null until a new successful result is selected.
|
||||||
|
|
||||||
### Generic Processing Artifacts
|
### Generic Processing Artifacts
|
||||||
|
|
||||||
@@ -185,6 +189,8 @@ EXECUTION_ATTEMPT ||--o{ PROCESSING_ARTIFACT : produces
|
|||||||
- Exactly one of `inline_payload` and `external_reference` is populated.
|
- Exactly one of `inline_payload` and `external_reference` is populated.
|
||||||
- Externally stored artifacts use application-managed relative references and are verified by SHA-256 and byte size.
|
- Externally stored artifacts use application-managed relative references and are verified by SHA-256 and byte size.
|
||||||
- Coordinate metadata declares units, origin, dimensions, and transformations when geometry is present.
|
- Coordinate metadata declares units, origin, dimensions, and transformations when geometry is present.
|
||||||
|
- Orientation-normalized binary model inputs and JSON quality-warning results use distinct versioned artifact types
|
||||||
|
and are attached to the exact consuming `EXECUTION_ATTEMPT`.
|
||||||
|
|
||||||
### Image Storage and Integrity
|
### Image Storage and Integrity
|
||||||
|
|
||||||
@@ -195,7 +201,7 @@ EXECUTION_ATTEMPT ||--o{ PROCESSING_ARTIFACT : produces
|
|||||||
### Page Ordering and Revisions
|
### Page Ordering and Revisions
|
||||||
|
|
||||||
- `SOURCE.page_number` dictates page ordering within a document.
|
- `SOURCE.page_number` dictates page ordering within a document.
|
||||||
- `SOURCE.raw_transcription` remains immutable machine output.
|
- `SOURCE.raw_transcription` changes only through first-success selection or explicit candidate promotion.
|
||||||
- `SOURCE.revised_text` stores human edits and is the preferred display value when present.
|
- `SOURCE.revised_text` stores human edits and is the preferred display value when present.
|
||||||
|
|
||||||
### Semantic Registry Governance
|
### Semantic Registry Governance
|
||||||
|
|||||||
@@ -6,9 +6,15 @@ Do not summarize. Do not paraphrase. Do not modernize style.
|
|||||||
|
|
||||||
## Output Contract
|
## Output Contract
|
||||||
- Return only the transcription text.
|
- Return only the transcription text.
|
||||||
|
- Begin with exactly one applicable body marker:
|
||||||
|
- `[document body handwritten]`
|
||||||
|
- `[document body typewritten]`
|
||||||
|
- `[document body typeset]`
|
||||||
|
- `[document body mixed]`
|
||||||
- Preserve original wording, punctuation, and meaningful structure.
|
- Preserve original wording, punctuation, and meaningful structure.
|
||||||
- Keep line/section flow readable while preserving intent and document organization.
|
- Keep line/section flow readable while preserving intent and document organization.
|
||||||
- Never invent missing content.
|
- Never invent missing content.
|
||||||
|
- Use ordinary plain-text characters rather than HTML entities.
|
||||||
|
|
||||||
## Rules for Ambiguous or Damaged Text
|
## Rules for Ambiguous or Damaged Text
|
||||||
|
|
||||||
@@ -46,16 +52,31 @@ Do not summarize. Do not paraphrase. Do not modernize style.
|
|||||||
- Signal location before the note text.
|
- Signal location before the note text.
|
||||||
- Example form: `[written in left margin: ...]`
|
- Example form: `[written in left margin: ...]`
|
||||||
|
|
||||||
### Printed and handwritten text
|
### Document body medium
|
||||||
|
- Use `[document body handwritten]` when the main body is written by hand.
|
||||||
|
- Use `[document body typewritten]` for mechanically typewritten pages. Uneven impressions,
|
||||||
|
monospaced characters, worn type, and other typewriter defects are not handwriting.
|
||||||
|
- Use `[document body typeset]` for printed pages composed with movable type or comparable
|
||||||
|
typesetting.
|
||||||
|
- Use `[document body mixed]` when substantial body content uses more than one medium, such
|
||||||
|
as a completed printed form.
|
||||||
- Preserve printed and handwritten text together in their original reading context.
|
- Preserve printed and handwritten text together in their original reading context.
|
||||||
- On mixed documents such as completed forms, leave printed labels and instructions unmarked.
|
- On mixed documents, leave printed labels and instructions unmarked and wrap only actual
|
||||||
- Wrap handwritten entries in `[handwritten: ...]`.
|
handwritten entries in `[handwritten: ...]`.
|
||||||
- Mark handwritten signatures as `[handwritten signature: ...]`.
|
- Mark handwritten signatures as `[handwritten signature: ...]`.
|
||||||
- If the main body is entirely handwritten, add `[document body handwritten]` once at the beginning rather than marking every line.
|
- If the main body is entirely handwritten, use its one body marker rather than wrapping
|
||||||
|
each line in `[handwritten: ...]`.
|
||||||
- Mark later notes or uncertain additions as `[handwritten annotation: ...]`.
|
- Mark later notes or uncertain additions as `[handwritten annotation: ...]`.
|
||||||
- When authorship is unclear, use `[handwritten annotation, author uncertain: ...]`.
|
- When authorship is unclear, use `[handwritten annotation, author uncertain: ...]`.
|
||||||
- Do not infer authorship, writing date, or whether different handwriting belongs to different people unless explicitly evident.
|
- Do not infer authorship, writing date, or whether different handwriting belongs to different people unless explicitly evident.
|
||||||
|
|
||||||
|
### Structured layouts
|
||||||
|
- Preserve tables of contents as associated title, dotted-leader, and page-reference rows.
|
||||||
|
- Preserve tables and forms in reading order while keeping labels associated with their values.
|
||||||
|
- Preserve columns in their evident reading order; do not interleave unrelated rows.
|
||||||
|
- Preserve captions with the visual element they describe.
|
||||||
|
- Preserve marginalia with its location marker and page numbers in their evident position.
|
||||||
|
|
||||||
### Line-break hyphenation
|
### Line-break hyphenation
|
||||||
- Rejoin words split across line breaks when they are clearly one word.
|
- Rejoin words split across line breaks when they are clearly one word.
|
||||||
- Remove only line-break hyphens used for wrapping.
|
- Remove only line-break hyphens used for wrapping.
|
||||||
@@ -80,4 +101,5 @@ Before finalizing, ensure:
|
|||||||
2. Uncertain/illegible areas are explicitly marked.
|
2. Uncertain/illegible areas are explicitly marked.
|
||||||
3. Crossed-out and inserted text are preserved with required tags.
|
3. Crossed-out and inserted text are preserved with required tags.
|
||||||
4. Structure/ordering is preserved as faithfully as possible.
|
4. Structure/ordering is preserved as faithfully as possible.
|
||||||
5. Handwriting is identified using the mixed-text conventions without separating it from its printed context.
|
5. Exactly one document-body marker appears, and localized handwriting markers are used only where applicable.
|
||||||
|
6. Tables, forms, columns, captions, marginalia, dotted leaders, and page references retain their associations.
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ dependencies = [
|
|||||||
"fastapi>=0.138.0",
|
"fastapi>=0.138.0",
|
||||||
"nicegui==3.13.0",
|
"nicegui==3.13.0",
|
||||||
"openrouter>=0.7.0",
|
"openrouter>=0.7.0",
|
||||||
|
"pillow>=10.0.0",
|
||||||
"psycopg2-binary>=2.9.12",
|
"psycopg2-binary>=2.9.12",
|
||||||
"pydantic>=2.13.4",
|
"pydantic>=2.13.4",
|
||||||
"pydantic-settings>=2.9.1",
|
"pydantic-settings>=2.9.1",
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ from pydantic import ConfigDict
|
|||||||
from pydantic import Field
|
from pydantic import Field
|
||||||
from pydantic import SecretStr
|
from pydantic import SecretStr
|
||||||
from pydantic import StringConstraints
|
from pydantic import StringConstraints
|
||||||
|
from pydantic import field_validator
|
||||||
|
from pydantic import model_validator
|
||||||
from pydantic_settings import BaseSettings
|
from pydantic_settings import BaseSettings
|
||||||
from pydantic_settings import SettingsConfigDict
|
from pydantic_settings import SettingsConfigDict
|
||||||
|
|
||||||
@@ -33,6 +35,7 @@ NonEmptyStr = Annotated[str, StringConstraints(strip_whitespace=True, min_length
|
|||||||
PromptFilename = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, pattern=r"^[^/\\]+$")]
|
PromptFilename = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, pattern=r"^[^/\\]+$")]
|
||||||
Probability = Annotated[float, Field(ge=0.0, le=1.0)]
|
Probability = Annotated[float, Field(ge=0.0, le=1.0)]
|
||||||
Temperature = Annotated[float, Field(ge=0.0, le=2.0)]
|
Temperature = Annotated[float, Field(ge=0.0, le=2.0)]
|
||||||
|
DEFAULT_PROVIDER_MODEL = "google/gemini-2.5-flash"
|
||||||
|
|
||||||
|
|
||||||
class SqliteSettings(BaseModel):
|
class SqliteSettings(BaseModel):
|
||||||
@@ -79,7 +82,8 @@ class Settings(BaseSettings):
|
|||||||
# --- AI provider ---
|
# --- AI provider ---
|
||||||
provider: Provider = Provider.OPENROUTER
|
provider: Provider = Provider.OPENROUTER
|
||||||
openrouter_api_key: SecretStr
|
openrouter_api_key: SecretStr
|
||||||
provider_model: NonEmptyStr | None = None
|
provider_model: NonEmptyStr | None = DEFAULT_PROVIDER_MODEL
|
||||||
|
provider_models: tuple[NonEmptyStr, ...] = ()
|
||||||
openrouter_http_referer: NonEmptyStr | None = None
|
openrouter_http_referer: NonEmptyStr | None = None
|
||||||
openrouter_app_title: NonEmptyStr | None = None
|
openrouter_app_title: NonEmptyStr | None = None
|
||||||
default_prompt_name: PromptFilename = "transcribe_document.md"
|
default_prompt_name: PromptFilename = "transcribe_document.md"
|
||||||
@@ -108,6 +112,31 @@ class Settings(BaseSettings):
|
|||||||
worker_min_transcription_lines: int = Field(default=0, ge=0)
|
worker_min_transcription_lines: int = Field(default=0, ge=0)
|
||||||
worker_fail_on_finish_reason_length: bool = False
|
worker_fail_on_finish_reason_length: bool = False
|
||||||
|
|
||||||
|
@field_validator("provider_models", mode="before")
|
||||||
|
@classmethod
|
||||||
|
def validate_provider_models_input(cls, value: object) -> object:
|
||||||
|
if value is None:
|
||||||
|
return ()
|
||||||
|
if isinstance(value, (list, tuple)) and not value:
|
||||||
|
raise ValueError("PROVIDER_MODELS must contain at least one model")
|
||||||
|
return value
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def normalize_provider_models(self) -> "Settings":
|
||||||
|
"""Build the immutable model selector with the configured default first."""
|
||||||
|
configured = self.provider_models
|
||||||
|
|
||||||
|
default_model = self.provider_model or DEFAULT_PROVIDER_MODEL
|
||||||
|
object.__setattr__(self, "provider_model", default_model)
|
||||||
|
ordered = (default_model, *configured)
|
||||||
|
deduplicated: list[str] = []
|
||||||
|
for model in ordered:
|
||||||
|
normalized = model.strip()
|
||||||
|
if normalized not in deduplicated:
|
||||||
|
deduplicated.append(normalized)
|
||||||
|
object.__setattr__(self, "provider_models", tuple(deduplicated))
|
||||||
|
return self
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def should_bootstrap_schema(self) -> bool:
|
def should_bootstrap_schema(self) -> bool:
|
||||||
"""Return whether startup should auto-create schema for this environment."""
|
"""Return whether startup should auto-create schema for this environment."""
|
||||||
|
|||||||
@@ -50,6 +50,11 @@ class JobSourceStatus(StrEnum):
|
|||||||
FAILED = "failed"
|
FAILED = "failed"
|
||||||
|
|
||||||
|
|
||||||
|
class JobPurpose(StrEnum):
|
||||||
|
TRANSCRIPTION = "transcription"
|
||||||
|
RETRANSCRIPTION = "retranscription"
|
||||||
|
|
||||||
|
|
||||||
class DocumentType(SQLModel, table=True):
|
class DocumentType(SQLModel, table=True):
|
||||||
"""Registry of allowed document types."""
|
"""Registry of allowed document types."""
|
||||||
|
|
||||||
@@ -180,6 +185,18 @@ class Job(SQLModel, table=True):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
retry_count: int = Field(default=0, ge=0)
|
retry_count: int = Field(default=0, ge=0)
|
||||||
|
purpose: JobPurpose = Field(
|
||||||
|
default=JobPurpose.TRANSCRIPTION,
|
||||||
|
sa_column=Column(
|
||||||
|
SAEnum(
|
||||||
|
JobPurpose,
|
||||||
|
values_callable=lambda enum_cls: [item.value for item in enum_cls],
|
||||||
|
native_enum=False,
|
||||||
|
),
|
||||||
|
nullable=False,
|
||||||
|
default=JobPurpose.TRANSCRIPTION.value,
|
||||||
|
),
|
||||||
|
)
|
||||||
date_created: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
date_created: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||||
date_updated: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
date_updated: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||||
provider: str | None = None
|
provider: str | None = None
|
||||||
@@ -240,6 +257,11 @@ class Source(SQLModel, table=True):
|
|||||||
file_hash: str
|
file_hash: str
|
||||||
file_size_bytes: int = Field(sa_column=Column(BigInteger(), nullable=False))
|
file_size_bytes: int = Field(sa_column=Column(BigInteger(), nullable=False))
|
||||||
raw_transcription: str | None = None
|
raw_transcription: str | None = None
|
||||||
|
preferred_execution_attempt_id: UUID | None = Field(
|
||||||
|
default=None,
|
||||||
|
foreign_key="execution_attempt.id",
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
revised_text: str | None = None
|
revised_text: str | None = None
|
||||||
date_uploaded: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
date_uploaded: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||||
date_revised: datetime | None = None
|
date_revised: datetime | None = None
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ async def create_all(*, engine: AsyncEngine | None = None) -> None:
|
|||||||
await connection.run_sync(SQLModel.metadata.create_all)
|
await connection.run_sync(SQLModel.metadata.create_all)
|
||||||
await _upgrade_person_family_search_id(connection)
|
await _upgrade_person_family_search_id(connection)
|
||||||
await _upgrade_v42_evidence_tables(connection)
|
await _upgrade_v42_evidence_tables(connection)
|
||||||
|
await _upgrade_v45_selection_columns(connection)
|
||||||
await seed_registry_defaults(engine=active_engine)
|
await seed_registry_defaults(engine=active_engine)
|
||||||
logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url)
|
logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url)
|
||||||
|
|
||||||
@@ -42,6 +43,7 @@ async def upgrade_schema(*, engine: AsyncEngine | None = None) -> None:
|
|||||||
async with active_engine.begin() as connection:
|
async with active_engine.begin() as connection:
|
||||||
await _upgrade_person_family_search_id(connection)
|
await _upgrade_person_family_search_id(connection)
|
||||||
await _upgrade_v42_evidence_tables(connection)
|
await _upgrade_v42_evidence_tables(connection)
|
||||||
|
await _upgrade_v45_selection_columns(connection)
|
||||||
|
|
||||||
|
|
||||||
async def _upgrade_v42_evidence_tables(connection: AsyncConnection) -> None:
|
async def _upgrade_v42_evidence_tables(connection: AsyncConnection) -> None:
|
||||||
@@ -54,6 +56,33 @@ async def _upgrade_v42_evidence_tables(connection: AsyncConnection) -> None:
|
|||||||
await connection.run_sync(create_tables)
|
await connection.run_sync(create_tables)
|
||||||
|
|
||||||
|
|
||||||
|
async def _upgrade_v45_selection_columns(connection: AsyncConnection) -> None:
|
||||||
|
"""Add V4.5 purpose and preferred-attempt provenance columns."""
|
||||||
|
|
||||||
|
def inspect_columns(sync_connection) -> tuple[set[str], set[str]]:
|
||||||
|
database = inspect(sync_connection)
|
||||||
|
tables = set(database.get_table_names())
|
||||||
|
job_columns = {column["name"] for column in database.get_columns("job")} if "job" in tables else set()
|
||||||
|
source_columns = (
|
||||||
|
{column["name"] for column in database.get_columns("source")} if "source" in tables else set()
|
||||||
|
)
|
||||||
|
return job_columns, source_columns
|
||||||
|
|
||||||
|
job_columns, source_columns = await connection.run_sync(inspect_columns)
|
||||||
|
if job_columns and "purpose" not in job_columns:
|
||||||
|
await connection.execute(
|
||||||
|
text("ALTER TABLE job ADD COLUMN purpose VARCHAR NOT NULL DEFAULT 'transcription'")
|
||||||
|
)
|
||||||
|
if source_columns and "preferred_execution_attempt_id" not in source_columns:
|
||||||
|
await connection.execute(text("ALTER TABLE source ADD COLUMN preferred_execution_attempt_id CHAR(32)"))
|
||||||
|
await connection.execute(
|
||||||
|
text(
|
||||||
|
"CREATE INDEX IF NOT EXISTS ix_source_preferred_execution_attempt_id "
|
||||||
|
"ON source (preferred_execution_attempt_id)"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _upgrade_person_family_search_id(connection: AsyncConnection) -> None:
|
async def _upgrade_person_family_search_id(connection: AsyncConnection) -> None:
|
||||||
"""Add the nullable V4.1 FamilySearch field to an existing database."""
|
"""Add the nullable V4.1 FamilySearch field to an existing database."""
|
||||||
|
|
||||||
|
|||||||
@@ -111,6 +111,7 @@ class TranscriptionProvider(Protocol):
|
|||||||
temperature: float | None = None,
|
temperature: float | None = None,
|
||||||
top_p: float | None = None,
|
top_p: float | None = None,
|
||||||
source_reference: SourceEvidenceReference | None = None,
|
source_reference: SourceEvidenceReference | None = None,
|
||||||
|
requested_model: str | None = None,
|
||||||
) -> TranscriptionResult:
|
) -> TranscriptionResult:
|
||||||
"""Transcribe the provided image according to the prompt text."""
|
"""Transcribe the provided image according to the prompt text."""
|
||||||
...
|
...
|
||||||
|
|||||||
@@ -232,6 +232,7 @@ class OpenRouterTranscriptionProvider:
|
|||||||
temperature: float | None = None,
|
temperature: float | None = None,
|
||||||
top_p: float | None = None,
|
top_p: float | None = None,
|
||||||
source_reference: SourceEvidenceReference | None = None,
|
source_reference: SourceEvidenceReference | None = None,
|
||||||
|
requested_model: str | None = None,
|
||||||
) -> TranscriptionResult:
|
) -> TranscriptionResult:
|
||||||
"""Send prompt + image to OpenRouter and return normalized text output."""
|
"""Send prompt + image to OpenRouter and return normalized text output."""
|
||||||
request = self._build_request(
|
request = self._build_request(
|
||||||
@@ -240,6 +241,7 @@ class OpenRouterTranscriptionProvider:
|
|||||||
mime_type=mime_type,
|
mime_type=mime_type,
|
||||||
temperature=temperature,
|
temperature=temperature,
|
||||||
top_p=top_p,
|
top_p=top_p,
|
||||||
|
requested_model=requested_model,
|
||||||
)
|
)
|
||||||
manifest = self._build_request_manifest(
|
manifest = self._build_request_manifest(
|
||||||
request=request,
|
request=request,
|
||||||
@@ -303,7 +305,7 @@ class OpenRouterTranscriptionProvider:
|
|||||||
transport_evidence=transport,
|
transport_evidence=transport,
|
||||||
failure_phase="response_validation",
|
failure_phase="response_validation",
|
||||||
) from exc
|
) from exc
|
||||||
model = validated_response.model or self.model
|
model = validated_response.model or requested_model or self.model
|
||||||
metadata = self._build_metadata(validated_response)
|
metadata = self._build_metadata(validated_response)
|
||||||
logger.info("OpenRouter transcription completed using model=%s", model)
|
logger.info("OpenRouter transcription completed using model=%s", model)
|
||||||
return TranscriptionResult(
|
return TranscriptionResult(
|
||||||
@@ -339,7 +341,7 @@ class OpenRouterTranscriptionProvider:
|
|||||||
omitted = tuple(name for name in ("temperature", "top_p") if name not in explicit)
|
omitted = tuple(name for name in ("temperature", "top_p") if name not in explicit)
|
||||||
return RequestManifest(
|
return RequestManifest(
|
||||||
provider="openrouter",
|
provider="openrouter",
|
||||||
requested_model=self.model,
|
requested_model=request.model,
|
||||||
request=JSON_OBJECT_ADAPTER.validate_python(sanitized_request),
|
request=JSON_OBJECT_ADAPTER.validate_python(sanitized_request),
|
||||||
source=source_reference,
|
source=source_reference,
|
||||||
explicitly_supplied_parameters=explicit,
|
explicitly_supplied_parameters=explicit,
|
||||||
@@ -481,6 +483,7 @@ class OpenRouterTranscriptionProvider:
|
|||||||
mime_type: str,
|
mime_type: str,
|
||||||
temperature: float | None,
|
temperature: float | None,
|
||||||
top_p: float | None,
|
top_p: float | None,
|
||||||
|
requested_model: str | None = None,
|
||||||
) -> OpenRouterRequest:
|
) -> OpenRouterRequest:
|
||||||
image_b64 = base64.b64encode(image_bytes).decode("ascii")
|
image_b64 = base64.b64encode(image_bytes).decode("ascii")
|
||||||
data_url = f"data:{mime_type};base64,{image_b64}"
|
data_url = f"data:{mime_type};base64,{image_b64}"
|
||||||
@@ -491,7 +494,7 @@ class OpenRouterTranscriptionProvider:
|
|||||||
media_content = ImageContent(image_url=ImageUrl(url=data_url))
|
media_content = ImageContent(image_url=ImageUrl(url=data_url))
|
||||||
|
|
||||||
return OpenRouterRequest(
|
return OpenRouterRequest(
|
||||||
model=self.model,
|
model=requested_model or self.model,
|
||||||
messages=(UserMessage(content=(TextContent(text=prompt_text), media_content)),),
|
messages=(UserMessage(content=(TextContent(text=prompt_text), media_content)),),
|
||||||
http_referer=self._settings.openrouter_http_referer,
|
http_referer=self._settings.openrouter_http_referer,
|
||||||
x_open_router_title=self._settings.openrouter_app_title,
|
x_open_router_title=self._settings.openrouter_app_title,
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
"""Metadata-directed orientation normalization for provider image input."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
from PIL import UnidentifiedImageError
|
||||||
|
|
||||||
|
from transcription.errors import AppError
|
||||||
|
from transcription.errors import ErrorCategory
|
||||||
|
|
||||||
|
ORIENTATION_TAG = 274
|
||||||
|
ORIENTATION_SCHEMA = "transcription.orientation-normalization"
|
||||||
|
ORIENTATION_SCHEMA_VERSION = "1"
|
||||||
|
ORIENTATION_PRODUCER = "transcription.orientation-normalizer"
|
||||||
|
ORIENTATION_PRODUCER_VERSION = "1"
|
||||||
|
|
||||||
|
_TRANSPOSE_BY_ORIENTATION = {
|
||||||
|
3: (Image.Transpose.ROTATE_180, 180),
|
||||||
|
6: (Image.Transpose.ROTATE_270, 90),
|
||||||
|
8: (Image.Transpose.ROTATE_90, 270),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class OrientationNormalizationError(AppError):
|
||||||
|
"""Raised when a supported raster image cannot be normalized safely."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class OrientationNormalization:
|
||||||
|
"""Exact derivative bytes and transformation metadata."""
|
||||||
|
|
||||||
|
content: bytes
|
||||||
|
media_type: str
|
||||||
|
suffix: str
|
||||||
|
original_orientation: int
|
||||||
|
applied_rotation_degrees: int
|
||||||
|
original_width: int
|
||||||
|
original_height: int
|
||||||
|
derivative_width: int
|
||||||
|
derivative_height: int
|
||||||
|
|
||||||
|
@property
|
||||||
|
def digest_sha256(self) -> str:
|
||||||
|
return hashlib.sha256(self.content).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_orientation(path: str | Path, *, media_type: str) -> OrientationNormalization | None:
|
||||||
|
"""Physically apply supported EXIF rotation, returning None for a safe no-op."""
|
||||||
|
source_path = Path(path)
|
||||||
|
if media_type not in {"image/jpeg", "image/png", "image/tiff"}:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
with Image.open(source_path) as image:
|
||||||
|
orientation = int(image.getexif().get(ORIENTATION_TAG, 1))
|
||||||
|
transformation = _TRANSPOSE_BY_ORIENTATION.get(orientation)
|
||||||
|
if transformation is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
transpose, rotation = transformation
|
||||||
|
if image.format == "TIFF":
|
||||||
|
original_width = int(image.tag_v2.get(256, image.width))
|
||||||
|
original_height = int(image.tag_v2.get(257, image.height))
|
||||||
|
# Pillow applies TIFF orientation while decoding; copying freezes those upright pixels.
|
||||||
|
normalized = image.copy()
|
||||||
|
else:
|
||||||
|
original_width, original_height = image.size
|
||||||
|
normalized = image.transpose(transpose)
|
||||||
|
output = io.BytesIO()
|
||||||
|
exif = normalized.getexif()
|
||||||
|
if ORIENTATION_TAG in exif:
|
||||||
|
del exif[ORIENTATION_TAG]
|
||||||
|
save_kwargs: dict[str, object] = {"format": image.format}
|
||||||
|
if image.format in {"JPEG", "PNG"}:
|
||||||
|
save_kwargs["exif"] = exif.tobytes()
|
||||||
|
if image.format == "JPEG":
|
||||||
|
save_kwargs.update({"quality": 95, "subsampling": 0})
|
||||||
|
normalized.save(output, **save_kwargs)
|
||||||
|
except (OSError, ValueError, UnidentifiedImageError) as exc:
|
||||||
|
raise OrientationNormalizationError(
|
||||||
|
f"Source image orientation could not be normalized: {source_path.name}",
|
||||||
|
category=ErrorCategory.VALIDATION,
|
||||||
|
suggestion="Verify that the curated Source is a valid supported raster image.",
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
suffix = source_path.suffix.lower()
|
||||||
|
return OrientationNormalization(
|
||||||
|
content=output.getvalue(),
|
||||||
|
media_type=media_type,
|
||||||
|
suffix=suffix,
|
||||||
|
original_orientation=orientation,
|
||||||
|
applied_rotation_degrees=rotation,
|
||||||
|
original_width=original_width,
|
||||||
|
original_height=original_height,
|
||||||
|
derivative_width=normalized.width,
|
||||||
|
derivative_height=normalized.height,
|
||||||
|
)
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
"""Deterministic, provider-neutral transcription quality warnings."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
|
from pydantic import BaseModel
|
||||||
|
from pydantic import ConfigDict
|
||||||
|
|
||||||
|
QUALITY_ANALYSIS_SCHEMA = "transcription.quality-warnings"
|
||||||
|
QUALITY_ANALYSIS_VERSION = "1"
|
||||||
|
QUALITY_ANALYSIS_PRODUCER = "transcription.quality"
|
||||||
|
QUALITY_ANALYSIS_PRODUCER_VERSION = "1"
|
||||||
|
|
||||||
|
_BODY_MARKER_RE = re.compile(
|
||||||
|
r"\[document body (?:handwritten|typewritten|typeset|mixed)\]",
|
||||||
|
flags=re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_HANDWRITTEN_LINE_RE = re.compile(r"(?m)^\s*\[handwritten:\s*.+\]\s*$", flags=re.IGNORECASE)
|
||||||
|
_HTML_ENTITY_RE = re.compile(r"&(?:#[0-9]{1,7}|#x[0-9a-f]{1,6}|[a-z][a-z0-9]{1,31});", flags=re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
|
class QualityWarningCode(StrEnum):
|
||||||
|
"""Stable identifiers for V4.5 output warning rules."""
|
||||||
|
|
||||||
|
REPLACEMENT_CHARACTER = "replacement_character"
|
||||||
|
MULTIPLE_BODY_MARKERS = "multiple_body_markers"
|
||||||
|
REDUNDANT_HANDWRITING_WRAPPERS = "redundant_handwriting_wrappers"
|
||||||
|
UNRESOLVED_HTML_ENTITY = "unresolved_html_entity"
|
||||||
|
|
||||||
|
|
||||||
|
class QualityWarning(BaseModel):
|
||||||
|
"""One immutable warning produced without changing transcription text."""
|
||||||
|
|
||||||
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||||
|
|
||||||
|
code: QualityWarningCode
|
||||||
|
detail: str
|
||||||
|
|
||||||
|
|
||||||
|
def analyze_transcription_quality(text: str) -> tuple[QualityWarning, ...]:
|
||||||
|
"""Return deterministic warnings in stable rule order."""
|
||||||
|
warnings: list[QualityWarning] = []
|
||||||
|
if "\ufffd" in text:
|
||||||
|
warnings.append(
|
||||||
|
QualityWarning(
|
||||||
|
code=QualityWarningCode.REPLACEMENT_CHARACTER,
|
||||||
|
detail="Transcription contains one or more Unicode replacement characters.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
body_markers = _BODY_MARKER_RE.findall(text)
|
||||||
|
if len(body_markers) > 1:
|
||||||
|
warnings.append(
|
||||||
|
QualityWarning(
|
||||||
|
code=QualityWarningCode.MULTIPLE_BODY_MARKERS,
|
||||||
|
detail=f"Transcription contains {len(body_markers)} document-body markers; exactly one is expected.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if re.search(r"\[document body handwritten\]", text, flags=re.IGNORECASE):
|
||||||
|
wrappers = _HANDWRITTEN_LINE_RE.findall(text)
|
||||||
|
if len(wrappers) > 1:
|
||||||
|
warnings.append(
|
||||||
|
QualityWarning(
|
||||||
|
code=QualityWarningCode.REDUNDANT_HANDWRITING_WRAPPERS,
|
||||||
|
detail=(
|
||||||
|
"A wholly handwritten document also uses repeated whole-line handwriting wrappers."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if _HTML_ENTITY_RE.search(text):
|
||||||
|
warnings.append(
|
||||||
|
QualityWarning(
|
||||||
|
code=QualityWarningCode.UNRESOLVED_HTML_ENTITY,
|
||||||
|
detail="Transcription contains a likely unresolved HTML entity.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(warnings)
|
||||||
|
|
||||||
|
|
||||||
|
def quality_warning_payload(warnings: tuple[QualityWarning, ...]) -> dict:
|
||||||
|
"""Build the versioned JSON artifact payload."""
|
||||||
|
return {
|
||||||
|
"schema_name": QUALITY_ANALYSIS_SCHEMA,
|
||||||
|
"schema_version": QUALITY_ANALYSIS_VERSION,
|
||||||
|
"warnings": [warning.model_dump(mode="json") for warning in warnings],
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import base64
|
import base64
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import inspect
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
@@ -51,6 +52,11 @@ from transcription.providers import get_transcription_provider
|
|||||||
from transcription.providers.evidence import canonical_json_bytes
|
from transcription.providers.evidence import canonical_json_bytes
|
||||||
|
|
||||||
from .base import ServiceBase
|
from .base import ServiceBase
|
||||||
|
from .normalization import ORIENTATION_PRODUCER
|
||||||
|
from .normalization import ORIENTATION_PRODUCER_VERSION
|
||||||
|
from .normalization import ORIENTATION_SCHEMA
|
||||||
|
from .normalization import ORIENTATION_SCHEMA_VERSION
|
||||||
|
from .normalization import normalize_orientation
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -96,6 +102,10 @@ class SourceDeleteBlockedError(TranscriptionError):
|
|||||||
"""Raised when source deletion is blocked by dependency policy."""
|
"""Raised when source deletion is blocked by dependency policy."""
|
||||||
|
|
||||||
|
|
||||||
|
class CandidatePromotionError(TranscriptionError):
|
||||||
|
"""Raised when a machine attempt cannot be selected for its Source."""
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class SourceNavigation:
|
class SourceNavigation:
|
||||||
"""Adjacent Source identifiers within one ordered Document."""
|
"""Adjacent Source identifiers within one ordered Document."""
|
||||||
@@ -104,6 +114,20 @@ class SourceNavigation:
|
|||||||
next_id: UUID | None
|
next_id: UUID | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ProviderInput:
|
||||||
|
"""Resolved immutable bytes and evidence identity for one provider request."""
|
||||||
|
|
||||||
|
path: Path
|
||||||
|
digest_sha256: str
|
||||||
|
byte_size: int
|
||||||
|
media_type: str
|
||||||
|
width: int | None = None
|
||||||
|
height: int | None = None
|
||||||
|
derivative_id: UUID | None = None
|
||||||
|
transformation: str | None = None
|
||||||
|
|
||||||
|
|
||||||
class SourceService(ServiceBase):
|
class SourceService(ServiceBase):
|
||||||
"""Manage source records, media payloads, revisions, and page execution output."""
|
"""Manage source records, media payloads, revisions, and page execution output."""
|
||||||
|
|
||||||
@@ -348,6 +372,30 @@ class SourceService(ServiceBase):
|
|||||||
)
|
)
|
||||||
return job_source
|
return job_source
|
||||||
|
|
||||||
|
async def read_job_source_for_job(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
job_id: UUID,
|
||||||
|
source_id: UUID,
|
||||||
|
session: AsyncSession | None = None,
|
||||||
|
) -> JobSource:
|
||||||
|
"""Read the unique JobSource association for one job and Source."""
|
||||||
|
async with self._session_scope(session) as _session:
|
||||||
|
job_source = (
|
||||||
|
await _session.exec(
|
||||||
|
select(JobSource)
|
||||||
|
.where(JobSource.job_id == job_id)
|
||||||
|
.where(JobSource.source_id == source_id)
|
||||||
|
)
|
||||||
|
).first()
|
||||||
|
if job_source is None:
|
||||||
|
raise TranscriptionNotFoundError(
|
||||||
|
f"Source {source_id} is not linked to Job {job_id}",
|
||||||
|
category=ErrorCategory.NOT_FOUND,
|
||||||
|
suggestion="Refresh the Job and retry.",
|
||||||
|
)
|
||||||
|
return job_source
|
||||||
|
|
||||||
async def update_job_source(self, job_source: JobSource, *, session: AsyncSession | None = None) -> JobSource:
|
async def update_job_source(self, job_source: JobSource, *, session: AsyncSession | None = None) -> JobSource:
|
||||||
"""Update an existing job_source record."""
|
"""Update an existing job_source record."""
|
||||||
async with self._session_scope(session) as _session:
|
async with self._session_scope(session) as _session:
|
||||||
@@ -472,6 +520,7 @@ class SourceService(ServiceBase):
|
|||||||
provider: str | None = None,
|
provider: str | None = None,
|
||||||
model: str | None = None,
|
model: str | None = None,
|
||||||
request_manifest: RequestManifest | None = None,
|
request_manifest: RequestManifest | None = None,
|
||||||
|
model_input_artifact_id: UUID | None = None,
|
||||||
transport_evidence: TransportEvidence | None = None,
|
transport_evidence: TransportEvidence | None = None,
|
||||||
failure_phase: str | None = None,
|
failure_phase: str | None = None,
|
||||||
error_category: str | None = None,
|
error_category: str | None = None,
|
||||||
@@ -511,9 +560,6 @@ class SourceService(ServiceBase):
|
|||||||
metadata_payload = _validate_transcription_metadata(ai_metadata)
|
metadata_payload = _validate_transcription_metadata(ai_metadata)
|
||||||
raw_response_payload = _validate_json_object(raw_api_response, field_name="raw_api_response")
|
raw_response_payload = _validate_json_object(raw_api_response, field_name="raw_api_response")
|
||||||
|
|
||||||
if text is not None:
|
|
||||||
source.raw_transcription = text
|
|
||||||
|
|
||||||
existing_job_source = await _session.exec(
|
existing_job_source = await _session.exec(
|
||||||
select(JobSource).where(JobSource.job_id == job_id).where(JobSource.source_id == source_id)
|
select(JobSource).where(JobSource.job_id == job_id).where(JobSource.source_id == source_id)
|
||||||
)
|
)
|
||||||
@@ -586,10 +632,76 @@ class SourceService(ServiceBase):
|
|||||||
else max(0, int((finish_time - start_time).total_seconds() * 1000)),
|
else max(0, int((finish_time - start_time).total_seconds() * 1000)),
|
||||||
)
|
)
|
||||||
_session.add(attempt)
|
_session.add(attempt)
|
||||||
|
await _session.flush()
|
||||||
|
|
||||||
|
manifest_derivative_id = (
|
||||||
|
request_manifest.source.derivative_id if request_manifest is not None else None
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
model_input_artifact_id is not None
|
||||||
|
and manifest_derivative_id is not None
|
||||||
|
and model_input_artifact_id != manifest_derivative_id
|
||||||
|
):
|
||||||
|
raise TranscriptionError(
|
||||||
|
"Provider-input artifact does not match the request manifest",
|
||||||
|
category=ErrorCategory.CONFLICT,
|
||||||
|
suggestion="Persist the exact normalized input consumed by this attempt.",
|
||||||
|
)
|
||||||
|
derivative_id = model_input_artifact_id or manifest_derivative_id
|
||||||
|
if derivative_id is not None:
|
||||||
|
derivative = await _session.get(ProcessingArtifact, derivative_id)
|
||||||
|
if derivative is None or derivative.source_id != source_id:
|
||||||
|
raise TranscriptionError(
|
||||||
|
"Provider-input derivative evidence is missing or belongs to another Source",
|
||||||
|
category=ErrorCategory.CONFLICT,
|
||||||
|
suggestion="Restore the normalized input artifact before persisting the attempt.",
|
||||||
|
)
|
||||||
|
derivative.execution_attempt_id = attempt.id
|
||||||
|
|
||||||
|
if (
|
||||||
|
text is not None
|
||||||
|
and source.raw_transcription is None
|
||||||
|
and source.preferred_execution_attempt_id is None
|
||||||
|
):
|
||||||
|
source.raw_transcription = text
|
||||||
|
source.preferred_execution_attempt_id = attempt.id
|
||||||
|
|
||||||
await self._finalize(session=_session, caller_session=session, refresh=(job, source, job_source, attempt))
|
await self._finalize(session=_session, caller_session=session, refresh=(job, source, job_source, attempt))
|
||||||
return job_source
|
return job_source
|
||||||
|
|
||||||
|
async def promote_machine_attempt(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
source_id: UUID,
|
||||||
|
execution_attempt_id: UUID,
|
||||||
|
session: AsyncSession | None = None,
|
||||||
|
) -> Source:
|
||||||
|
"""Atomically select one successful machine attempt as the Source projection."""
|
||||||
|
async with self._session_scope(session) as _session:
|
||||||
|
source = await _session.get(Source, source_id)
|
||||||
|
if source is None:
|
||||||
|
raise TranscriptionNotFoundError(
|
||||||
|
f"Source with id {source_id} not found",
|
||||||
|
category=ErrorCategory.NOT_FOUND,
|
||||||
|
suggestion="Refresh Source Detail and retry.",
|
||||||
|
)
|
||||||
|
attempt = await _session.get(ExecutionAttempt, execution_attempt_id)
|
||||||
|
if (
|
||||||
|
attempt is None
|
||||||
|
or attempt.source_id != source_id
|
||||||
|
or attempt.status != JobSourceStatus.TRANSCRIBED
|
||||||
|
or not attempt.raw_transcription
|
||||||
|
):
|
||||||
|
raise CandidatePromotionError(
|
||||||
|
"Only a successful transcription attempt belonging to this Source can be selected",
|
||||||
|
category=ErrorCategory.VALIDATION,
|
||||||
|
suggestion="Select an available successful candidate from Source Detail.",
|
||||||
|
)
|
||||||
|
source.preferred_execution_attempt_id = attempt.id
|
||||||
|
source.raw_transcription = attempt.raw_transcription
|
||||||
|
await self._finalize(session=_session, caller_session=session, refresh=(source,))
|
||||||
|
return source
|
||||||
|
|
||||||
async def list_execution_attempts(
|
async def list_execution_attempts(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -689,6 +801,110 @@ class SourceService(ServiceBase):
|
|||||||
external_path.unlink(missing_ok=True)
|
external_path.unlink(missing_ok=True)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
async def create_binary_artifact(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
source_id: UUID,
|
||||||
|
artifact_type: str,
|
||||||
|
media_type: str,
|
||||||
|
schema_name: str,
|
||||||
|
schema_version: str,
|
||||||
|
producer: str,
|
||||||
|
producer_version: str,
|
||||||
|
content: bytes,
|
||||||
|
suffix: str,
|
||||||
|
coordinate_metadata: dict[str, JsonValue] | None = None,
|
||||||
|
session: AsyncSession | None = None,
|
||||||
|
) -> ProcessingArtifact:
|
||||||
|
"""Persist exact binary derivative bytes in application-managed artifact storage."""
|
||||||
|
artifact_id = uuid4()
|
||||||
|
safe_suffix = suffix if suffix.startswith(".") and suffix[1:].isalnum() else ".bin"
|
||||||
|
relative_path = Path(str(source_id)) / f"{artifact_id}{safe_suffix.lower()}"
|
||||||
|
external_path = self.settings.artifact_dir / relative_path
|
||||||
|
self._write_external_artifact(path=external_path, content=content)
|
||||||
|
artifact = ProcessingArtifact(
|
||||||
|
id=artifact_id,
|
||||||
|
source_id=source_id,
|
||||||
|
artifact_type=artifact_type,
|
||||||
|
media_type=media_type,
|
||||||
|
schema_name=schema_name,
|
||||||
|
schema_version=schema_version,
|
||||||
|
producer=producer,
|
||||||
|
producer_version=producer_version,
|
||||||
|
external_reference=relative_path.as_posix(),
|
||||||
|
payload_sha256=hashlib.sha256(content).hexdigest(),
|
||||||
|
byte_size=len(content),
|
||||||
|
coordinate_metadata=coordinate_metadata,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return await self.create_processing_artifact(artifact, session=session)
|
||||||
|
except Exception:
|
||||||
|
external_path.unlink(missing_ok=True)
|
||||||
|
raise
|
||||||
|
|
||||||
|
async def resolve_provider_input(
|
||||||
|
self,
|
||||||
|
source: Source,
|
||||||
|
*,
|
||||||
|
session: AsyncSession | None = None,
|
||||||
|
) -> ProviderInput:
|
||||||
|
"""Resolve original or physically orientation-normalized provider input."""
|
||||||
|
media_type = source_mime_type(source.file_path)
|
||||||
|
normalized = normalize_orientation(source.file_path, media_type=media_type)
|
||||||
|
if normalized is None:
|
||||||
|
return ProviderInput(
|
||||||
|
path=Path(source.file_path),
|
||||||
|
digest_sha256=source.file_hash.lower(),
|
||||||
|
byte_size=source.file_size_bytes,
|
||||||
|
media_type=media_type,
|
||||||
|
)
|
||||||
|
|
||||||
|
metadata: dict[str, JsonValue] = {
|
||||||
|
"units": "pixels",
|
||||||
|
"origin": "top-left",
|
||||||
|
"width": normalized.derivative_width,
|
||||||
|
"height": normalized.derivative_height,
|
||||||
|
"transformations": [f"rotate-{normalized.applied_rotation_degrees}-degrees-clockwise"],
|
||||||
|
"source_id": str(source.id),
|
||||||
|
"original_digest_sha256": source.file_hash.lower(),
|
||||||
|
"original_byte_size": source.file_size_bytes,
|
||||||
|
"original_orientation": normalized.original_orientation,
|
||||||
|
"applied_rotation_degrees": normalized.applied_rotation_degrees,
|
||||||
|
"original_width": normalized.original_width,
|
||||||
|
"original_height": normalized.original_height,
|
||||||
|
"derivative_width": normalized.derivative_width,
|
||||||
|
"derivative_height": normalized.derivative_height,
|
||||||
|
"derivative_digest_sha256": normalized.digest_sha256,
|
||||||
|
"derivative_byte_size": len(normalized.content),
|
||||||
|
"original_media_type": media_type,
|
||||||
|
"derivative_media_type": normalized.media_type,
|
||||||
|
}
|
||||||
|
artifact = await self.create_binary_artifact(
|
||||||
|
source_id=source.id,
|
||||||
|
artifact_type="orientation_normalized_model_input",
|
||||||
|
media_type=normalized.media_type,
|
||||||
|
schema_name=ORIENTATION_SCHEMA,
|
||||||
|
schema_version=ORIENTATION_SCHEMA_VERSION,
|
||||||
|
producer=ORIENTATION_PRODUCER,
|
||||||
|
producer_version=ORIENTATION_PRODUCER_VERSION,
|
||||||
|
content=normalized.content,
|
||||||
|
suffix=normalized.suffix,
|
||||||
|
coordinate_metadata=metadata,
|
||||||
|
session=session,
|
||||||
|
)
|
||||||
|
if artifact.external_reference is None:
|
||||||
|
raise RuntimeError("Orientation derivative did not receive an external reference")
|
||||||
|
return ProviderInput(
|
||||||
|
path=self.settings.artifact_dir / artifact.external_reference,
|
||||||
|
digest_sha256=artifact.payload_sha256,
|
||||||
|
byte_size=artifact.byte_size,
|
||||||
|
media_type=artifact.media_type,
|
||||||
|
width=normalized.derivative_width,
|
||||||
|
height=normalized.derivative_height,
|
||||||
|
derivative_id=artifact.id,
|
||||||
|
transformation=f"{ORIENTATION_SCHEMA}@{ORIENTATION_SCHEMA_VERSION}",
|
||||||
|
)
|
||||||
|
|
||||||
def _write_external_artifact(self, *, path: Path, content: bytes) -> None:
|
def _write_external_artifact(self, *, path: Path, content: bytes) -> None:
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
temporary_path = path.with_suffix(f"{path.suffix}.tmp")
|
temporary_path = path.with_suffix(f"{path.suffix}.tmp")
|
||||||
@@ -986,6 +1202,7 @@ async def transcribe_document_image(
|
|||||||
settings: Settings | None = None,
|
settings: Settings | None = None,
|
||||||
provider: TranscriptionProvider | None = None,
|
provider: TranscriptionProvider | None = None,
|
||||||
source_reference: SourceEvidenceReference | None = None,
|
source_reference: SourceEvidenceReference | None = None,
|
||||||
|
requested_model: str | None = None,
|
||||||
) -> TranscriptionResult:
|
) -> TranscriptionResult:
|
||||||
"""Transcribe a local image using the configured prompt and provider."""
|
"""Transcribe a local image using the configured prompt and provider."""
|
||||||
runtime_settings = settings or get_settings()
|
runtime_settings = settings or get_settings()
|
||||||
@@ -1009,14 +1226,17 @@ async def transcribe_document_image(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
with handle_transcription_errors():
|
with handle_transcription_errors():
|
||||||
result = await adapter.transcribe(
|
transcribe_kwargs = {
|
||||||
prompt_text=prompt_execution.user_prompt,
|
"prompt_text": prompt_execution.user_prompt,
|
||||||
image_bytes=image_bytes,
|
"image_bytes": image_bytes,
|
||||||
mime_type=mime_type,
|
"mime_type": mime_type,
|
||||||
temperature=prompt_execution.temperature,
|
"temperature": prompt_execution.temperature,
|
||||||
top_p=prompt_execution.top_p,
|
"top_p": prompt_execution.top_p,
|
||||||
source_reference=source_reference,
|
"source_reference": source_reference,
|
||||||
)
|
}
|
||||||
|
if "requested_model" in inspect.signature(adapter.transcribe).parameters:
|
||||||
|
transcribe_kwargs["requested_model"] = requested_model
|
||||||
|
result = await adapter.transcribe(**transcribe_kwargs)
|
||||||
finally:
|
finally:
|
||||||
if owns_adapter:
|
if owns_adapter:
|
||||||
close = getattr(adapter, "aclose", None)
|
close = getattr(adapter, "aclose", None)
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import logging
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import UTC
|
from datetime import UTC
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||||
|
|
||||||
@@ -11,6 +12,8 @@ from ..config import Settings
|
|||||||
from ..config import get_settings
|
from ..config import get_settings
|
||||||
from ..db.models import Document
|
from ..db.models import Document
|
||||||
from ..db.models import Job
|
from ..db.models import Job
|
||||||
|
from ..db.models import JobPurpose
|
||||||
|
from ..db.models import JobSource
|
||||||
from ..db.models import JobSourceStatus
|
from ..db.models import JobSourceStatus
|
||||||
from ..db.models import JobStatus
|
from ..db.models import JobStatus
|
||||||
from ..db.models import Source
|
from ..db.models import Source
|
||||||
@@ -29,10 +32,15 @@ from . import ServiceBundle
|
|||||||
from .documents import DocumentService
|
from .documents import DocumentService
|
||||||
from .people import DocumentPersonInput
|
from .people import DocumentPersonInput
|
||||||
from .people import PeopleService
|
from .people import PeopleService
|
||||||
|
from .quality import QUALITY_ANALYSIS_PRODUCER
|
||||||
|
from .quality import QUALITY_ANALYSIS_PRODUCER_VERSION
|
||||||
|
from .quality import QUALITY_ANALYSIS_SCHEMA
|
||||||
|
from .quality import QUALITY_ANALYSIS_VERSION
|
||||||
|
from .quality import analyze_transcription_quality
|
||||||
|
from .quality import quality_warning_payload
|
||||||
from .sources import PromptExecution
|
from .sources import PromptExecution
|
||||||
from .sources import build_prompt_execution
|
from .sources import build_prompt_execution
|
||||||
from .sources import hash_prompt_text
|
from .sources import hash_prompt_text
|
||||||
from .sources import source_mime_type
|
|
||||||
from .sources import transcribe_document_image
|
from .sources import transcribe_document_image
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -66,6 +74,46 @@ async def update_document_with_people(
|
|||||||
return updated
|
return updated
|
||||||
|
|
||||||
|
|
||||||
|
async def create_source_retranscription_job(
|
||||||
|
*,
|
||||||
|
source_id,
|
||||||
|
model: str,
|
||||||
|
services: ServiceBundle,
|
||||||
|
settings: Settings | None = None,
|
||||||
|
) -> Job:
|
||||||
|
"""Create one immutable queued retranscription Job for an existing Source."""
|
||||||
|
runtime_settings = settings or get_settings()
|
||||||
|
if model not in runtime_settings.provider_models:
|
||||||
|
raise AppError(
|
||||||
|
f"Model is not configured for retranscription: {model}",
|
||||||
|
category=ErrorCategory.VALIDATION,
|
||||||
|
suggestion="Select one of the configured provider models.",
|
||||||
|
)
|
||||||
|
prompt = build_prompt_execution(settings=runtime_settings)
|
||||||
|
async with transaction_scope(session_factory=services.jobs.session_factory) as session:
|
||||||
|
source = await services.sources.read_source(source_id, session=session)
|
||||||
|
job = await services.jobs.create_job(
|
||||||
|
Job(
|
||||||
|
document_id=source.document_id,
|
||||||
|
purpose=JobPurpose.RETRANSCRIPTION,
|
||||||
|
provider=runtime_settings.provider.value,
|
||||||
|
model=model,
|
||||||
|
prompt_name=prompt.prompt_name,
|
||||||
|
prompt_hash=prompt.prompt_hash,
|
||||||
|
system_prompt=prompt.system_prompt,
|
||||||
|
user_prompt=prompt.user_prompt,
|
||||||
|
temperature=prompt.temperature,
|
||||||
|
top_p=prompt.top_p,
|
||||||
|
),
|
||||||
|
session=session,
|
||||||
|
)
|
||||||
|
await services.sources.create_job_source(
|
||||||
|
JobSource(job_id=job.id, source_id=source.id),
|
||||||
|
session=session,
|
||||||
|
)
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class _SuccessfulPage:
|
class _SuccessfulPage:
|
||||||
source: Source
|
source: Source
|
||||||
@@ -73,6 +121,7 @@ class _SuccessfulPage:
|
|||||||
started_at: datetime
|
started_at: datetime
|
||||||
finished_at: datetime
|
finished_at: datetime
|
||||||
duration_ms: int
|
duration_ms: int
|
||||||
|
model_input_artifact_id: UUID | None = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -89,6 +138,7 @@ class _FailedPage:
|
|||||||
normalized_metadata: dict | None = None
|
normalized_metadata: dict | None = None
|
||||||
provider: str | None = None
|
provider: str | None = None
|
||||||
model: str | None = None
|
model: str | None = None
|
||||||
|
model_input_artifact_id: UUID | None = None
|
||||||
|
|
||||||
|
|
||||||
async def advance_job(
|
async def advance_job(
|
||||||
@@ -168,22 +218,31 @@ async def process_queued_job( # noqa: PLR0915
|
|||||||
started_at = datetime.now(UTC)
|
started_at = datetime.now(UTC)
|
||||||
monotonic_started_at = asyncio.get_running_loop().time()
|
monotonic_started_at = asyncio.get_running_loop().time()
|
||||||
result: TranscriptionResult | None = None
|
result: TranscriptionResult | None = None
|
||||||
|
provider_input = None
|
||||||
page_outcome: _SuccessfulPage | _FailedPage
|
page_outcome: _SuccessfulPage | _FailedPage
|
||||||
try:
|
try:
|
||||||
|
provider_input = await services.sources.resolve_provider_input(source, session=session)
|
||||||
|
if session is not None:
|
||||||
|
await session.commit()
|
||||||
source_reference = SourceEvidenceReference(
|
source_reference = SourceEvidenceReference(
|
||||||
source_id=source.id,
|
source_id=source.id,
|
||||||
digest_sha256=source.file_hash.lower(),
|
digest_sha256=provider_input.digest_sha256,
|
||||||
byte_size=source.file_size_bytes,
|
byte_size=provider_input.byte_size,
|
||||||
media_type=source_mime_type(source.file_path),
|
media_type=provider_input.media_type,
|
||||||
page_number=source.page_number,
|
page_number=source.page_number,
|
||||||
|
width=provider_input.width,
|
||||||
|
height=provider_input.height,
|
||||||
|
derivative_id=provider_input.derivative_id,
|
||||||
|
transformation=provider_input.transformation,
|
||||||
)
|
)
|
||||||
result = await asyncio.wait_for(
|
result = await asyncio.wait_for(
|
||||||
_call_transcriber(
|
_call_transcriber(
|
||||||
source=source,
|
input_path=provider_input.path,
|
||||||
prompt_execution=prompt_execution,
|
prompt_execution=prompt_execution,
|
||||||
settings=runtime_settings,
|
settings=runtime_settings,
|
||||||
provider=services.sources.provider,
|
provider=services.sources.provider,
|
||||||
source_reference=source_reference,
|
source_reference=source_reference,
|
||||||
|
requested_model=source_job.model,
|
||||||
),
|
),
|
||||||
timeout=runtime_settings.worker_provider_timeout_seconds,
|
timeout=runtime_settings.worker_provider_timeout_seconds,
|
||||||
)
|
)
|
||||||
@@ -215,6 +274,7 @@ async def process_queued_job( # noqa: PLR0915
|
|||||||
started_at=started_at,
|
started_at=started_at,
|
||||||
finished_at=finished_at,
|
finished_at=finished_at,
|
||||||
duration_ms=max(0, int(elapsed_seconds * 1000)),
|
duration_ms=max(0, int(elapsed_seconds * 1000)),
|
||||||
|
model_input_artifact_id=provider_input.derivative_id,
|
||||||
)
|
)
|
||||||
successful_pages.append(page_outcome)
|
successful_pages.append(page_outcome)
|
||||||
except TimeoutError:
|
except TimeoutError:
|
||||||
@@ -241,6 +301,9 @@ async def process_queued_job( # noqa: PLR0915
|
|||||||
None,
|
None,
|
||||||
),
|
),
|
||||||
failure_phase="local_timeout",
|
failure_phase="local_timeout",
|
||||||
|
model_input_artifact_id=(
|
||||||
|
provider_input.derivative_id if provider_input is not None else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
failed_pages.append(page_outcome)
|
failed_pages.append(page_outcome)
|
||||||
logger.error(
|
logger.error(
|
||||||
@@ -295,6 +358,9 @@ async def process_queued_job( # noqa: PLR0915
|
|||||||
normalized_metadata=result.metadata_payload() if result is not None else None,
|
normalized_metadata=result.metadata_payload() if result is not None else None,
|
||||||
provider=result.provider if result is not None else None,
|
provider=result.provider if result is not None else None,
|
||||||
model=result.model if result is not None else None,
|
model=result.model if result is not None else None,
|
||||||
|
model_input_artifact_id=(
|
||||||
|
provider_input.derivative_id if provider_input is not None else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
failed_pages.append(page_outcome)
|
failed_pages.append(page_outcome)
|
||||||
logger.error(
|
logger.error(
|
||||||
@@ -470,12 +536,36 @@ async def _write_page_outcome(
|
|||||||
provider=result.provider,
|
provider=result.provider,
|
||||||
model=result.model,
|
model=result.model,
|
||||||
request_manifest=result.request_manifest,
|
request_manifest=result.request_manifest,
|
||||||
|
model_input_artifact_id=page.model_input_artifact_id,
|
||||||
transport_evidence=result.transport_evidence,
|
transport_evidence=result.transport_evidence,
|
||||||
started_at=page.started_at,
|
started_at=page.started_at,
|
||||||
finished_at=page.finished_at,
|
finished_at=page.finished_at,
|
||||||
duration_ms=page.duration_ms,
|
duration_ms=page.duration_ms,
|
||||||
session=session,
|
session=session,
|
||||||
)
|
)
|
||||||
|
job_source = await services.sources.read_job_source_for_job(
|
||||||
|
job_id=job.id,
|
||||||
|
source_id=source.id,
|
||||||
|
session=session,
|
||||||
|
)
|
||||||
|
attempt = await services.sources.read_latest_execution_attempt(
|
||||||
|
job_source_id=job_source.id,
|
||||||
|
session=session,
|
||||||
|
)
|
||||||
|
if attempt is None:
|
||||||
|
raise RuntimeError("Successful transcription did not create execution evidence")
|
||||||
|
warnings = analyze_transcription_quality(result.text)
|
||||||
|
await services.sources.create_json_artifact(
|
||||||
|
source_id=source.id,
|
||||||
|
execution_attempt_id=attempt.id,
|
||||||
|
artifact_type="transcription_quality_warnings",
|
||||||
|
schema_name=QUALITY_ANALYSIS_SCHEMA,
|
||||||
|
schema_version=QUALITY_ANALYSIS_VERSION,
|
||||||
|
producer=QUALITY_ANALYSIS_PRODUCER,
|
||||||
|
producer_version=QUALITY_ANALYSIS_PRODUCER_VERSION,
|
||||||
|
payload=quality_warning_payload(warnings),
|
||||||
|
session=session,
|
||||||
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
await services.sources.update_job_source_transcription(
|
await services.sources.update_job_source_transcription(
|
||||||
@@ -488,6 +578,7 @@ async def _write_page_outcome(
|
|||||||
provider=page.provider,
|
provider=page.provider,
|
||||||
model=page.model,
|
model=page.model,
|
||||||
request_manifest=page.request_manifest,
|
request_manifest=page.request_manifest,
|
||||||
|
model_input_artifact_id=page.model_input_artifact_id,
|
||||||
transport_evidence=page.transport_evidence,
|
transport_evidence=page.transport_evidence,
|
||||||
failure_phase=page.failure_phase,
|
failure_phase=page.failure_phase,
|
||||||
error_category=page.error.category.value,
|
error_category=page.error.category.value,
|
||||||
@@ -572,16 +663,17 @@ def _find_provider_error(exc: BaseException) -> ProviderError | None:
|
|||||||
|
|
||||||
async def _call_transcriber(
|
async def _call_transcriber(
|
||||||
*,
|
*,
|
||||||
source: Source,
|
input_path,
|
||||||
prompt_execution: PromptExecution,
|
prompt_execution: PromptExecution,
|
||||||
settings: Settings,
|
settings: Settings,
|
||||||
provider: TranscriptionProvider,
|
provider: TranscriptionProvider,
|
||||||
source_reference: SourceEvidenceReference,
|
source_reference: SourceEvidenceReference,
|
||||||
|
requested_model: str | None,
|
||||||
) -> TranscriptionResult:
|
) -> TranscriptionResult:
|
||||||
"""Call the current transcriber while supporting legacy injected test doubles."""
|
"""Call the current transcriber while supporting legacy injected test doubles."""
|
||||||
if "source_reference" in inspect.signature(transcribe_document_image).parameters:
|
if "source_reference" in inspect.signature(transcribe_document_image).parameters:
|
||||||
return await transcribe_document_image(
|
return await transcribe_document_image(
|
||||||
source.file_path,
|
input_path,
|
||||||
prompt_name=prompt_execution.prompt_name,
|
prompt_name=prompt_execution.prompt_name,
|
||||||
prompt_text=prompt_execution.user_prompt,
|
prompt_text=prompt_execution.user_prompt,
|
||||||
temperature=prompt_execution.temperature,
|
temperature=prompt_execution.temperature,
|
||||||
@@ -589,9 +681,10 @@ async def _call_transcriber(
|
|||||||
settings=settings,
|
settings=settings,
|
||||||
provider=provider,
|
provider=provider,
|
||||||
source_reference=source_reference,
|
source_reference=source_reference,
|
||||||
|
requested_model=requested_model,
|
||||||
)
|
)
|
||||||
return await transcribe_document_image(
|
return await transcribe_document_image(
|
||||||
source.file_path,
|
input_path,
|
||||||
prompt_name=prompt_execution.prompt_name,
|
prompt_name=prompt_execution.prompt_name,
|
||||||
prompt_text=prompt_execution.user_prompt,
|
prompt_text=prompt_execution.user_prompt,
|
||||||
temperature=prompt_execution.temperature,
|
temperature=prompt_execution.temperature,
|
||||||
|
|||||||
@@ -9,16 +9,21 @@ from uuid import UUID
|
|||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
from nicegui import ui
|
from nicegui import ui
|
||||||
|
|
||||||
|
from transcription.config import Settings
|
||||||
|
from transcription.config import get_settings
|
||||||
from transcription.db.models import Job
|
from transcription.db.models import Job
|
||||||
from transcription.db.models import JobSourceStatus
|
from transcription.db.models import JobSourceStatus
|
||||||
from transcription.db.models import JobStatus
|
from transcription.db.models import JobStatus
|
||||||
from transcription.db.session import session_scope
|
from transcription.db.session import session_scope
|
||||||
|
from transcription.services import ServiceBundle
|
||||||
from transcription.services.documents import DocumentService
|
from transcription.services.documents import DocumentService
|
||||||
from transcription.services.jobs import JobCancelBlockedError
|
from transcription.services.jobs import JobCancelBlockedError
|
||||||
from transcription.services.jobs import JobDeleteBlockedError
|
from transcription.services.jobs import JobDeleteBlockedError
|
||||||
from transcription.services.jobs import JobResubmitBlockedError
|
from transcription.services.jobs import JobResubmitBlockedError
|
||||||
from transcription.services.jobs import JobService
|
from transcription.services.jobs import JobService
|
||||||
|
from transcription.services.sources import SourceService
|
||||||
from transcription.services.store import create_job_for_document
|
from transcription.services.store import create_job_for_document
|
||||||
|
from transcription.services.workflows import create_source_retranscription_job
|
||||||
from transcription.ui.components.app_shell import render_navigation_header
|
from transcription.ui.components.app_shell import render_navigation_header
|
||||||
from transcription.ui.components.cards import archival_card
|
from transcription.ui.components.cards import archival_card
|
||||||
from transcription.ui.components.data_display import archival_badge
|
from transcription.ui.components.data_display import archival_badge
|
||||||
@@ -70,8 +75,13 @@ def register_page() -> None: # noqa: PLR0915
|
|||||||
await render_table()
|
await render_table()
|
||||||
|
|
||||||
@ui.page("/jobs/new")
|
@ui.page("/jobs/new")
|
||||||
async def job_create_page(request: Request, session_factory: SessionFactoryDep) -> None:
|
async def job_create_page( # noqa: PLR0915
|
||||||
|
request: Request,
|
||||||
|
session_factory: SessionFactoryDep,
|
||||||
|
) -> None:
|
||||||
documents_service = DocumentService(session_factory=session_factory)
|
documents_service = DocumentService(session_factory=session_factory)
|
||||||
|
sources_service = SourceService(session_factory=session_factory)
|
||||||
|
settings = _resolve_runtime_settings(request)
|
||||||
render_navigation_header(current_path="/jobs")
|
render_navigation_header(current_path="/jobs")
|
||||||
|
|
||||||
with ui.column().classes("w-full max-w-4xl mx-auto p-4 gap-4"):
|
with ui.column().classes("w-full max-w-4xl mx-auto p-4 gap-4"):
|
||||||
@@ -79,6 +89,16 @@ def register_page() -> None: # noqa: PLR0915
|
|||||||
"Create Processing Job", subtitle="Queue source files for AI transcription and entity processing."
|
"Create Processing Job", subtitle="Queue source files for AI transcription and entity processing."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
requested_source_id = _parse_uuid(request.query_params.get("source_id"))
|
||||||
|
try:
|
||||||
|
locked_source = (
|
||||||
|
await sources_service.read_source_detail(requested_source_id)
|
||||||
|
if requested_source_id is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
show_error(exc, title="Source unavailable", operation="jobs.retranscribe.load")
|
||||||
|
return
|
||||||
documents = await documents_service.list_documents()
|
documents = await documents_service.list_documents()
|
||||||
if not documents:
|
if not documents:
|
||||||
_render_no_documents_card()
|
_render_no_documents_card()
|
||||||
@@ -88,24 +108,67 @@ def register_page() -> None: # noqa: PLR0915
|
|||||||
|
|
||||||
with archival_card(extra_classes="gap-3"):
|
with archival_card(extra_classes="gap-3"):
|
||||||
document_options = {str(doc.id): doc.name for doc in documents}
|
document_options = {str(doc.id): doc.name for doc in documents}
|
||||||
|
if locked_source is not None:
|
||||||
|
ui.label(f"Source: {locked_source.upload_name} ({locked_source.id})").classes(
|
||||||
|
"text-sm ui-text-primary"
|
||||||
|
)
|
||||||
|
document_label = (
|
||||||
|
locked_source.document.name if locked_source.document else str(locked_source.document_id)
|
||||||
|
)
|
||||||
|
ui.label(f"Document: {document_label}").classes("text-sm ui-text-primary")
|
||||||
|
document_select = None
|
||||||
|
else:
|
||||||
document_select = (
|
document_select = (
|
||||||
ui.select(document_options, label="Target Document")
|
ui.select(document_options, label="Target Document")
|
||||||
.props("outlined")
|
.props("outlined")
|
||||||
.classes("w-full ui-form-surface")
|
.classes("w-full ui-form-surface")
|
||||||
)
|
)
|
||||||
|
|
||||||
requested_document_id = request.query_params.get("document_id")
|
requested_document_id = request.query_params.get("document_id")
|
||||||
if requested_document_id in document_options:
|
if requested_document_id in document_options:
|
||||||
document_select.value = requested_document_id
|
document_select.value = requested_document_id
|
||||||
|
|
||||||
with ui.row().classes("w-full gap-3 grid grid-cols-1 md:grid-cols-2"):
|
with ui.row().classes("w-full gap-3 grid grid-cols-1 md:grid-cols-2"):
|
||||||
|
if locked_source is not None:
|
||||||
|
provider_input = (
|
||||||
|
ui.input(label="Provider", value=settings.provider.value)
|
||||||
|
.props("outlined readonly")
|
||||||
|
.classes("ui-form-surface")
|
||||||
|
)
|
||||||
|
model_input = (
|
||||||
|
ui.select(list(settings.provider_models), label="Model", value=settings.provider_model)
|
||||||
|
.props("outlined")
|
||||||
|
.classes("ui-form-surface")
|
||||||
|
)
|
||||||
|
else:
|
||||||
provider_input = ui.input(label="Provider").props("outlined").classes("ui-form-surface")
|
provider_input = ui.input(label="Provider").props("outlined").classes("ui-form-surface")
|
||||||
model_input = ui.input(label="Model").props("outlined").classes("ui-form-surface")
|
model_input = ui.input(label="Model").props("outlined").classes("ui-form-surface")
|
||||||
|
|
||||||
|
if locked_source is None:
|
||||||
_render_upload_section(uploaded_files)
|
_render_upload_section(uploaded_files)
|
||||||
|
|
||||||
async def submit_create() -> None:
|
async def submit_create() -> None:
|
||||||
if not document_select.value:
|
if locked_source is not None:
|
||||||
|
try:
|
||||||
|
services = ServiceBundle(
|
||||||
|
documents=documents_service,
|
||||||
|
jobs=JobService(session_factory=session_factory, settings=settings),
|
||||||
|
sources=SourceService(session_factory=session_factory, settings=settings),
|
||||||
|
)
|
||||||
|
result_job = await create_source_retranscription_job(
|
||||||
|
source_id=locked_source.id,
|
||||||
|
model=str(model_input.value),
|
||||||
|
services=services,
|
||||||
|
settings=settings,
|
||||||
|
)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
show_error(exc, title="Create job failed", operation="jobs.retranscribe")
|
||||||
|
return
|
||||||
|
resolve_worker_notifier(request.app.state).notify()
|
||||||
|
ui.notify(f"Created retranscription job {result_job.id}", type="positive")
|
||||||
|
ui.navigate.to(f"/jobs/{result_job.id}")
|
||||||
|
return
|
||||||
|
|
||||||
|
if document_select is None or not document_select.value:
|
||||||
ui.notify("Document is required.", type="warning")
|
ui.notify("Document is required.", type="warning")
|
||||||
return
|
return
|
||||||
if not uploaded_files:
|
if not uploaded_files:
|
||||||
@@ -501,5 +564,12 @@ def _parse_uuid(value: str | None) -> UUID | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_runtime_settings(request: Request) -> Settings:
|
||||||
|
app_settings = getattr(request.app.state, "settings", None)
|
||||||
|
if isinstance(app_settings, Settings):
|
||||||
|
return app_settings
|
||||||
|
return get_settings()
|
||||||
|
|
||||||
|
|
||||||
def _latest_prompt_name(job: Job) -> str | None:
|
def _latest_prompt_name(job: Job) -> str | None:
|
||||||
return job.prompt_name
|
return job.prompt_name
|
||||||
|
|||||||
@@ -128,6 +128,7 @@ def register_page() -> None: # noqa: PLR0915
|
|||||||
source_artifacts = list(
|
source_artifacts = list(
|
||||||
await sources_service.list_processing_artifact_summaries(source_id=parsed_source_id)
|
await sources_service.list_processing_artifact_summaries(source_id=parsed_source_id)
|
||||||
)
|
)
|
||||||
|
attempts = list(await sources_service.list_execution_attempts(source_id=parsed_source_id))
|
||||||
except TranscriptionNotFoundError:
|
except TranscriptionNotFoundError:
|
||||||
ui.label("Source not found").classes("text-h6 ui-text-danger p-4")
|
ui.label("Source not found").classes("text-h6 ui-text-danger p-4")
|
||||||
return
|
return
|
||||||
@@ -149,6 +150,11 @@ def register_page() -> None: # noqa: PLR0915
|
|||||||
on_click=lambda: ui.navigate.to("/sources"),
|
on_click=lambda: ui.navigate.to("/sources"),
|
||||||
icon="arrow_back",
|
icon="arrow_back",
|
||||||
).props("flat")
|
).props("flat")
|
||||||
|
ui.button(
|
||||||
|
"Retranscribe Source",
|
||||||
|
on_click=lambda: ui.navigate.to(f"/jobs/new?source_id={source.id}"),
|
||||||
|
icon="refresh",
|
||||||
|
).classes("ui-btn-primary")
|
||||||
ui.button(
|
ui.button(
|
||||||
"Export Evidence",
|
"Export Evidence",
|
||||||
on_click=lambda: _download_evidence(
|
on_click=lambda: _download_evidence(
|
||||||
@@ -163,6 +169,11 @@ def register_page() -> None: # noqa: PLR0915
|
|||||||
icon="delete",
|
icon="delete",
|
||||||
extra_classes="text-xs",
|
extra_classes="text-xs",
|
||||||
)
|
)
|
||||||
|
if any(
|
||||||
|
artifact.artifact_type == "orientation_normalized_model_input"
|
||||||
|
for artifact in source_artifacts
|
||||||
|
):
|
||||||
|
archival_badge("Orientation normalized")
|
||||||
|
|
||||||
with ui.grid().classes("w-full grid-cols-12 gap-4"):
|
with ui.grid().classes("w-full grid-cols-12 gap-4"):
|
||||||
with ui.column().classes("col-span-12 lg:col-span-4 gap-2"):
|
with ui.column().classes("col-span-12 lg:col-span-4 gap-2"):
|
||||||
@@ -178,6 +189,11 @@ def register_page() -> None: # noqa: PLR0915
|
|||||||
latest_job_source=latest_job_source,
|
latest_job_source=latest_job_source,
|
||||||
sources_service=sources_service,
|
sources_service=sources_service,
|
||||||
)
|
)
|
||||||
|
_render_machine_candidates(
|
||||||
|
source=source,
|
||||||
|
attempts=attempts,
|
||||||
|
sources_service=sources_service,
|
||||||
|
)
|
||||||
_render_source_metadata_column(
|
_render_source_metadata_column(
|
||||||
source=source,
|
source=source,
|
||||||
latest_job_source=latest_job_source,
|
latest_job_source=latest_job_source,
|
||||||
@@ -552,6 +568,129 @@ def _render_source_transcription_zone(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _render_machine_candidates(
|
||||||
|
*,
|
||||||
|
source: Source,
|
||||||
|
attempts: list[ExecutionAttempt],
|
||||||
|
sources_service: SourceService,
|
||||||
|
) -> None:
|
||||||
|
successful = [
|
||||||
|
attempt
|
||||||
|
for attempt in attempts
|
||||||
|
if attempt.status.value == "transcribed" and attempt.raw_transcription
|
||||||
|
]
|
||||||
|
candidates = [
|
||||||
|
attempt for attempt in successful if attempt.id != source.preferred_execution_attempt_id
|
||||||
|
]
|
||||||
|
preferred_attempt = next(
|
||||||
|
(
|
||||||
|
attempt
|
||||||
|
for attempt in successful
|
||||||
|
if attempt.id == source.preferred_execution_attempt_id
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
with ui.column().classes("col-span-12 lg:col-span-8 gap-2"): # noqa: PLR1702, SIM117
|
||||||
|
with archival_card(title="Candidate Machine Transcriptions"):
|
||||||
|
if preferred_attempt is not None:
|
||||||
|
_render_attempt_warnings(preferred_attempt, label="Preferred machine transcription warnings")
|
||||||
|
if not source.raw_transcription:
|
||||||
|
render_empty_state("No successful machine transcription exists yet.", italic=True)
|
||||||
|
return
|
||||||
|
if not candidates:
|
||||||
|
render_empty_state("No candidate machine transcriptions are available.", italic=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
if source.revised_text:
|
||||||
|
ui.label(
|
||||||
|
"Selecting a machine candidate does not change the saved human revision used for display and print."
|
||||||
|
).classes("text-xs ui-text-muted")
|
||||||
|
|
||||||
|
for attempt in sorted(candidates, key=lambda item: item.created_at, reverse=True):
|
||||||
|
warning_count = _attempt_warning_count(attempt)
|
||||||
|
title = (
|
||||||
|
f"{attempt.created_at.isoformat()} | {attempt.provider} | "
|
||||||
|
f"{attempt.model or 'unknown model'} | Job {attempt.job_id}"
|
||||||
|
)
|
||||||
|
with ui.expansion(title, icon="warning" if warning_count else "compare").classes(
|
||||||
|
"w-full ui-row-surface"
|
||||||
|
):
|
||||||
|
if warning_count:
|
||||||
|
_render_attempt_warnings(attempt, label="Candidate warnings")
|
||||||
|
with ui.grid().classes("w-full grid-cols-1 md:grid-cols-2 gap-3"):
|
||||||
|
with ui.column().classes("gap-1"):
|
||||||
|
ui.label("Preferred machine transcription").classes("text-xs font-semibold")
|
||||||
|
ui.label(source.raw_transcription).classes(
|
||||||
|
"p-2 ui-note-box text-xs whitespace-pre-wrap"
|
||||||
|
)
|
||||||
|
with ui.column().classes("gap-1"):
|
||||||
|
ui.label("Candidate transcription").classes("text-xs font-semibold")
|
||||||
|
ui.label(attempt.raw_transcription or "").classes(
|
||||||
|
"p-2 ui-note-box text-xs whitespace-pre-wrap"
|
||||||
|
)
|
||||||
|
|
||||||
|
async def promote(candidate_id: UUID = attempt.id) -> None:
|
||||||
|
try:
|
||||||
|
await sources_service.promote_machine_attempt(
|
||||||
|
source_id=source.id,
|
||||||
|
execution_attempt_id=candidate_id,
|
||||||
|
)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
show_error(exc, title="Promotion failed", operation="sources.candidate.promote")
|
||||||
|
return
|
||||||
|
ui.notify("Preferred machine transcription updated", type="positive")
|
||||||
|
ui.navigate.to(f"/sources/{source.id}")
|
||||||
|
|
||||||
|
with ui.dialog() as confirmation, ui.card():
|
||||||
|
ui.label("Use this machine transcription?")
|
||||||
|
ui.label("All earlier attempts and any human revision will be preserved.").classes(
|
||||||
|
"text-xs ui-text-muted"
|
||||||
|
)
|
||||||
|
|
||||||
|
async def confirm_promotion(callback=promote) -> None:
|
||||||
|
confirmation.close()
|
||||||
|
await callback()
|
||||||
|
|
||||||
|
with ui.row().classes("justify-end gap-2"):
|
||||||
|
ui.button("Cancel", on_click=confirmation.close).props("flat")
|
||||||
|
ui.button(
|
||||||
|
"Use this transcription",
|
||||||
|
on_click=confirm_promotion,
|
||||||
|
).classes("ui-btn-primary")
|
||||||
|
ui.button(
|
||||||
|
"Use this transcription",
|
||||||
|
on_click=confirmation.open,
|
||||||
|
icon="check_circle",
|
||||||
|
).classes("ui-btn-primary")
|
||||||
|
|
||||||
|
|
||||||
|
def _attempt_warning_count(attempt: ExecutionAttempt) -> int:
|
||||||
|
return len(_attempt_warnings(attempt))
|
||||||
|
|
||||||
|
|
||||||
|
def _attempt_warnings(attempt: ExecutionAttempt) -> list[dict[str, object]]:
|
||||||
|
for artifact in attempt.artifacts:
|
||||||
|
if artifact.artifact_type != "transcription_quality_warnings" or artifact.inline_payload is None:
|
||||||
|
continue
|
||||||
|
warnings = artifact.inline_payload.get("warnings")
|
||||||
|
if isinstance(warnings, list):
|
||||||
|
return [warning for warning in warnings if isinstance(warning, dict)]
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _render_attempt_warnings(attempt: ExecutionAttempt, *, label: str) -> None:
|
||||||
|
warnings = _attempt_warnings(attempt)
|
||||||
|
if not warnings:
|
||||||
|
return
|
||||||
|
with ui.column().classes("w-full gap-1 p-2 ui-note-box"):
|
||||||
|
ui.label(label).classes("text-xs font-semibold ui-text-danger")
|
||||||
|
for warning in warnings:
|
||||||
|
detail = warning.get("detail")
|
||||||
|
code = warning.get("code")
|
||||||
|
ui.label(f"{code}: {detail}").classes("text-xs ui-text-danger")
|
||||||
|
|
||||||
|
|
||||||
def _reset_revision_text(revision_input: ui.textarea, source: Source, original_transcription: str | None) -> None:
|
def _reset_revision_text(revision_input: ui.textarea, source: Source, original_transcription: str | None) -> None:
|
||||||
fallback_text = source.revised_text if source.revised_text is not None else (original_transcription or "")
|
fallback_text = source.revised_text if source.revised_text is not None else (original_transcription or "")
|
||||||
revision_input.value = fallback_text
|
revision_input.value = fallback_text
|
||||||
@@ -564,7 +703,7 @@ def _latest_job_source(source: Source) -> JobSource | None:
|
|||||||
|
|
||||||
|
|
||||||
def _resolve_original_transcription(*, source: Source, latest_job_source: JobSource | None) -> str | None:
|
def _resolve_original_transcription(*, source: Source, latest_job_source: JobSource | None) -> str | None:
|
||||||
if latest_job_source is not None and latest_job_source.raw_transcription:
|
if source.raw_transcription is None and latest_job_source is not None:
|
||||||
return latest_job_source.raw_transcription
|
return latest_job_source.raw_transcription
|
||||||
return source.raw_transcription
|
return source.raw_transcription
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
"""Integration tests for end-to-end upload and worker pipeline behavior."""
|
"""Integration tests for end-to-end upload and worker pipeline behavior."""
|
||||||
|
|
||||||
|
import io
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
from transcription.config import Settings
|
from transcription.config import Settings
|
||||||
from transcription.db.models import Document
|
from transcription.db.models import Document
|
||||||
@@ -18,6 +20,12 @@ from transcription.services.store import create_job_for_document
|
|||||||
from transcription.services.workflows import advance_job
|
from transcription.services.workflows import advance_job
|
||||||
|
|
||||||
|
|
||||||
|
def _jpeg_bytes(color: str = "white") -> bytes:
|
||||||
|
output = io.BytesIO()
|
||||||
|
Image.new("RGB", (2, 2), color=color).save(output, format="JPEG")
|
||||||
|
return output.getvalue()
|
||||||
|
|
||||||
|
|
||||||
def _build_services(default_session_factory) -> ServiceBundle:
|
def _build_services(default_session_factory) -> ServiceBundle:
|
||||||
services = ServiceBundle()
|
services = ServiceBundle()
|
||||||
object.__setattr__(
|
object.__setattr__(
|
||||||
@@ -56,7 +64,7 @@ class TestPipelineSuccessFlow:
|
|||||||
)
|
)
|
||||||
upload_result = await create_document_job(
|
upload_result = await create_document_job(
|
||||||
filename="pipeline.jpg",
|
filename="pipeline.jpg",
|
||||||
file_bytes=b"pipeline-bytes",
|
file_bytes=_jpeg_bytes(),
|
||||||
session=async_session,
|
session=async_session,
|
||||||
settings=settings,
|
settings=settings,
|
||||||
)
|
)
|
||||||
@@ -141,9 +149,9 @@ class TestPipelineSuccessFlow:
|
|||||||
create_result = await create_job_for_document(
|
create_result = await create_job_for_document(
|
||||||
document_id=document.id,
|
document_id=document.id,
|
||||||
source_files=[
|
source_files=[
|
||||||
("page-01.jpg", b"one"),
|
("page-01.jpg", _jpeg_bytes("white")),
|
||||||
("page-02.jpg", b"two"),
|
("page-02.jpg", _jpeg_bytes("gray")),
|
||||||
("page-03.jpg", b"three"),
|
("page-03.jpg", _jpeg_bytes("black")),
|
||||||
],
|
],
|
||||||
session=async_session,
|
session=async_session,
|
||||||
settings=settings,
|
settings=settings,
|
||||||
@@ -206,8 +214,8 @@ class TestPipelineSuccessFlow:
|
|||||||
create_result = await create_job_for_document(
|
create_result = await create_job_for_document(
|
||||||
document_id=document.id,
|
document_id=document.id,
|
||||||
source_files=[
|
source_files=[
|
||||||
("page-01.jpg", b"one"),
|
("page-01.jpg", _jpeg_bytes("white")),
|
||||||
("page-02.jpg", b"two"),
|
("page-02.jpg", _jpeg_bytes("gray")),
|
||||||
],
|
],
|
||||||
session=async_session,
|
session=async_session,
|
||||||
settings=settings,
|
settings=settings,
|
||||||
@@ -272,8 +280,8 @@ class TestPipelineSuccessFlow:
|
|||||||
create_result = await create_job_for_document(
|
create_result = await create_job_for_document(
|
||||||
document_id=document.id,
|
document_id=document.id,
|
||||||
source_files=[
|
source_files=[
|
||||||
("page-01.jpg", b"one"),
|
("page-01.jpg", _jpeg_bytes("white")),
|
||||||
("page-02.jpg", b"two"),
|
("page-02.jpg", _jpeg_bytes("gray")),
|
||||||
],
|
],
|
||||||
session=async_session,
|
session=async_session,
|
||||||
settings=settings,
|
settings=settings,
|
||||||
@@ -346,7 +354,7 @@ class TestPipelineFailureFlow:
|
|||||||
settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
|
settings = Settings(openrouter_api_key="test-key", upload_dir=tmp_path)
|
||||||
upload_result = await create_document_job(
|
upload_result = await create_document_job(
|
||||||
filename="pipeline.jpg",
|
filename="pipeline.jpg",
|
||||||
file_bytes=b"pipeline-bytes",
|
file_bytes=_jpeg_bytes(),
|
||||||
session=async_session,
|
session=async_session,
|
||||||
settings=settings,
|
settings=settings,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,228 @@
|
|||||||
|
"""Tests for V4.5 metadata-directed orientation normalization."""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from transcription.config import Settings
|
||||||
|
from transcription.db.models import Document
|
||||||
|
from transcription.db.models import Job
|
||||||
|
from transcription.db.models import JobSource
|
||||||
|
from transcription.db.models import Source
|
||||||
|
from transcription.providers import RequestManifest
|
||||||
|
from transcription.providers import TranscriptionResult
|
||||||
|
from transcription.providers.evidence import build_software_context
|
||||||
|
from transcription.services import ServiceBundle
|
||||||
|
from transcription.services.documents import DocumentService
|
||||||
|
from transcription.services.jobs import JobService
|
||||||
|
from transcription.services.normalization import normalize_orientation
|
||||||
|
from transcription.services.sources import SourceService
|
||||||
|
from transcription.services.workflows import process_queued_job
|
||||||
|
|
||||||
|
|
||||||
|
def _write_oriented_jpeg(path: Path, *, orientation: int) -> bytes:
|
||||||
|
image = Image.new("RGB", (2, 3))
|
||||||
|
image.putdata(
|
||||||
|
[
|
||||||
|
(255, 0, 0),
|
||||||
|
(255, 0, 0),
|
||||||
|
(0, 255, 0),
|
||||||
|
(0, 255, 0),
|
||||||
|
(0, 0, 255),
|
||||||
|
(0, 0, 255),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
exif = Image.Exif()
|
||||||
|
exif[274] = orientation
|
||||||
|
image.save(path, format="JPEG", quality=100, subsampling=0, exif=exif)
|
||||||
|
return path.read_bytes()
|
||||||
|
|
||||||
|
|
||||||
|
def _write_oriented_image(path: Path, *, orientation: int, image_format: str) -> None:
|
||||||
|
image = Image.new("RGB", (2, 3), color="white")
|
||||||
|
exif = Image.Exif()
|
||||||
|
exif[274] = orientation
|
||||||
|
image.save(path, format=image_format, exif=exif)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_orientation_three_is_physically_rotated_and_metadata_removed(tmp_path):
|
||||||
|
path = tmp_path / "upside-down.jpg"
|
||||||
|
original = _write_oriented_jpeg(path, orientation=3)
|
||||||
|
|
||||||
|
result = normalize_orientation(path, media_type="image/jpeg")
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result.applied_rotation_degrees == 180
|
||||||
|
assert path.read_bytes() == original
|
||||||
|
with Image.open(path) as source_image, Image.open(io.BytesIO(result.content)) as derivative:
|
||||||
|
assert source_image.getexif()[274] == 3
|
||||||
|
assert derivative.getexif().get(274, 1) == 1
|
||||||
|
assert derivative.getpixel((0, 0))[2] > derivative.getpixel((0, 0))[0]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_orientation_one_is_noop(tmp_path):
|
||||||
|
path = tmp_path / "upright.jpg"
|
||||||
|
_write_oriented_jpeg(path, orientation=1)
|
||||||
|
|
||||||
|
assert normalize_orientation(path, media_type="image/jpeg") is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("filename", "image_format", "media_type"),
|
||||||
|
[
|
||||||
|
("oriented.png", "PNG", "image/png"),
|
||||||
|
("oriented.tiff", "TIFF", "image/tiff"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_supported_non_jpeg_orientation_is_normalized(
|
||||||
|
tmp_path,
|
||||||
|
filename,
|
||||||
|
image_format,
|
||||||
|
media_type,
|
||||||
|
):
|
||||||
|
path = tmp_path / filename
|
||||||
|
_write_oriented_image(path, orientation=6, image_format=image_format)
|
||||||
|
|
||||||
|
result = normalize_orientation(path, media_type=media_type)
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result.applied_rotation_degrees == 90
|
||||||
|
assert (result.derivative_width, result.derivative_height) == (3, 2)
|
||||||
|
with Image.open(io.BytesIO(result.content)) as derivative:
|
||||||
|
assert derivative.getexif().get(274, 1) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_resolve_provider_input_persists_exact_derivative(default_session_factory, tmp_path):
|
||||||
|
source_path = tmp_path / "source.jpg"
|
||||||
|
original = _write_oriented_jpeg(source_path, orientation=3)
|
||||||
|
settings = Settings(
|
||||||
|
openrouter_api_key="test-key",
|
||||||
|
artifact_dir=tmp_path / "artifacts",
|
||||||
|
provider_models=None,
|
||||||
|
)
|
||||||
|
documents = DocumentService(session_factory=default_session_factory, settings=settings)
|
||||||
|
sources = SourceService(session_factory=default_session_factory, settings=settings)
|
||||||
|
document = await documents.create_document(Document(name="Oriented"))
|
||||||
|
source = await sources.create_source(
|
||||||
|
Source(
|
||||||
|
document_id=document.id,
|
||||||
|
page_number=1,
|
||||||
|
upload_name="source.jpg",
|
||||||
|
filename="source.jpg",
|
||||||
|
file_path=str(source_path),
|
||||||
|
file_hash=hashlib.sha256(original).hexdigest(),
|
||||||
|
file_size_bytes=len(original),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
provider_input = await sources.resolve_provider_input(source)
|
||||||
|
artifacts = await sources.list_processing_artifacts(source_id=source.id)
|
||||||
|
|
||||||
|
assert source_path.read_bytes() == original
|
||||||
|
assert provider_input.derivative_id == artifacts[0].id
|
||||||
|
assert provider_input.path.read_bytes() != original
|
||||||
|
assert hashlib.sha256(provider_input.path.read_bytes()).hexdigest() == provider_input.digest_sha256
|
||||||
|
assert artifacts[0].coordinate_metadata["original_orientation"] == 3
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_worker_sends_exact_derivative_and_links_attempt_evidence(
|
||||||
|
default_session_factory,
|
||||||
|
tmp_path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
source_path = tmp_path / "source.jpg"
|
||||||
|
original = _write_oriented_jpeg(source_path, orientation=3)
|
||||||
|
prompt_dir = tmp_path / "prompts"
|
||||||
|
prompt_dir.mkdir()
|
||||||
|
(prompt_dir / "transcribe_document.md").write_text("Transcribe verbatim.", encoding="utf-8")
|
||||||
|
settings = Settings(
|
||||||
|
openrouter_api_key="test-key",
|
||||||
|
artifact_dir=tmp_path / "artifacts",
|
||||||
|
prompt_dir=prompt_dir,
|
||||||
|
provider_models=None,
|
||||||
|
)
|
||||||
|
services = ServiceBundle(
|
||||||
|
documents=DocumentService(session_factory=default_session_factory, settings=settings),
|
||||||
|
jobs=JobService(session_factory=default_session_factory, settings=settings),
|
||||||
|
sources=SourceService(session_factory=default_session_factory, settings=settings),
|
||||||
|
)
|
||||||
|
document = await services.documents.create_document(Document(name="Pipeline"))
|
||||||
|
source = await services.sources.create_source(
|
||||||
|
Source(
|
||||||
|
document_id=document.id,
|
||||||
|
page_number=1,
|
||||||
|
upload_name="source.jpg",
|
||||||
|
filename="source.jpg",
|
||||||
|
file_path=str(source_path),
|
||||||
|
file_hash=hashlib.sha256(original).hexdigest(),
|
||||||
|
file_size_bytes=len(original),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
job = await services.jobs.create_job(Job(document_id=document.id))
|
||||||
|
await services.sources.create_job_source(JobSource(job_id=job.id, source_id=source.id))
|
||||||
|
loaded = await services.jobs.read_job(job.id)
|
||||||
|
captured: dict[str, object] = {}
|
||||||
|
|
||||||
|
async def fake_transcribe(
|
||||||
|
image_path,
|
||||||
|
*,
|
||||||
|
prompt_name,
|
||||||
|
prompt_text,
|
||||||
|
temperature,
|
||||||
|
top_p,
|
||||||
|
settings,
|
||||||
|
provider,
|
||||||
|
source_reference,
|
||||||
|
requested_model,
|
||||||
|
):
|
||||||
|
_ = (prompt_name, temperature, top_p, settings, provider, requested_model)
|
||||||
|
image_bytes = Path(image_path).read_bytes()
|
||||||
|
captured["bytes"] = image_bytes
|
||||||
|
captured["source_reference"] = source_reference
|
||||||
|
manifest = RequestManifest(
|
||||||
|
provider="fixture",
|
||||||
|
requested_model="fixture/model",
|
||||||
|
request={"model": "fixture/model"},
|
||||||
|
source=source_reference,
|
||||||
|
optional_parameter_states={"temperature": "omitted", "top_p": "omitted"},
|
||||||
|
prompt_content=prompt_text,
|
||||||
|
prompt_sha256=hashlib.sha256(prompt_text.encode()).hexdigest(),
|
||||||
|
timeout_seconds=20,
|
||||||
|
retry_policy="none",
|
||||||
|
software=build_software_context(
|
||||||
|
adapter_name="fixture",
|
||||||
|
adapter_version="1",
|
||||||
|
client_library="transcription",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return TranscriptionResult(
|
||||||
|
text="[document body typewritten]\nDamaged \ufffd text",
|
||||||
|
provider="fixture",
|
||||||
|
model="fixture/model",
|
||||||
|
request_manifest=manifest,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr("transcription.services.workflows.transcribe_document_image", fake_transcribe)
|
||||||
|
|
||||||
|
await process_queued_job(job=loaded, services=services, settings=settings)
|
||||||
|
|
||||||
|
attempts = await services.sources.list_execution_attempts(source_id=source.id)
|
||||||
|
artifacts = await services.sources.list_processing_artifacts(source_id=source.id)
|
||||||
|
source_reference = captured["source_reference"]
|
||||||
|
assert source_path.read_bytes() == original
|
||||||
|
assert hashlib.sha256(captured["bytes"]).hexdigest() == source_reference.digest_sha256
|
||||||
|
assert source_reference.derivative_id is not None
|
||||||
|
assert {artifact.artifact_type for artifact in artifacts} == {
|
||||||
|
"orientation_normalized_model_input",
|
||||||
|
"transcription_quality_warnings",
|
||||||
|
}
|
||||||
|
assert {artifact.execution_attempt_id for artifact in artifacts} == {attempts[0].id}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
"""Tests for deterministic V4.5 transcription warnings."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from transcription.services.quality import QualityWarningCode
|
||||||
|
from transcription.services.quality import analyze_transcription_quality
|
||||||
|
from transcription.services.quality import quality_warning_payload
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_quality_analysis_reports_each_v45_warning_without_mutating_text():
|
||||||
|
text = (
|
||||||
|
"[document body handwritten]\n"
|
||||||
|
"[document body typewritten]\n"
|
||||||
|
"[handwritten: first line]\n"
|
||||||
|
"[handwritten: second line]\n"
|
||||||
|
"Damaged \ufffd text &"
|
||||||
|
)
|
||||||
|
|
||||||
|
warnings = analyze_transcription_quality(text)
|
||||||
|
|
||||||
|
assert [warning.code for warning in warnings] == [
|
||||||
|
QualityWarningCode.REPLACEMENT_CHARACTER,
|
||||||
|
QualityWarningCode.MULTIPLE_BODY_MARKERS,
|
||||||
|
QualityWarningCode.REDUNDANT_HANDWRITING_WRAPPERS,
|
||||||
|
QualityWarningCode.UNRESOLVED_HTML_ENTITY,
|
||||||
|
]
|
||||||
|
assert text.endswith("&")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_quality_analysis_accepts_clean_transcription():
|
||||||
|
assert analyze_transcription_quality("[document body typewritten]\nClean text.") == ()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_quality_warning_payload_is_versioned():
|
||||||
|
payload = quality_warning_payload(
|
||||||
|
analyze_transcription_quality("[document body typeset]\nBroken \ufffd")
|
||||||
|
)
|
||||||
|
|
||||||
|
assert payload["schema_name"] == "transcription.quality-warnings"
|
||||||
|
assert payload["schema_version"] == "1"
|
||||||
|
assert payload["warnings"][0]["code"] == "replacement_character"
|
||||||
@@ -0,0 +1,130 @@
|
|||||||
|
"""V4.5 retranscription candidate and promotion tests."""
|
||||||
|
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from transcription.config import Settings
|
||||||
|
from transcription.db.models import Document
|
||||||
|
from transcription.db.models import Job
|
||||||
|
from transcription.db.models import JobPurpose
|
||||||
|
from transcription.db.models import JobSource
|
||||||
|
from transcription.db.models import Source
|
||||||
|
from transcription.services import ServiceBundle
|
||||||
|
from transcription.services.documents import DocumentService
|
||||||
|
from transcription.services.jobs import JobService
|
||||||
|
from transcription.services.sources import CandidatePromotionError
|
||||||
|
from transcription.services.sources import SourceService
|
||||||
|
from transcription.services.workflows import create_source_retranscription_job
|
||||||
|
|
||||||
|
|
||||||
|
def _services(default_session_factory, settings: Settings) -> ServiceBundle:
|
||||||
|
return ServiceBundle(
|
||||||
|
documents=DocumentService(session_factory=default_session_factory, settings=settings),
|
||||||
|
jobs=JobService(session_factory=default_session_factory, settings=settings),
|
||||||
|
sources=SourceService(session_factory=default_session_factory, settings=settings),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _seed_source(services: ServiceBundle) -> Source:
|
||||||
|
document = await services.documents.create_document(Document(name="V4.5 source"))
|
||||||
|
source = Source(
|
||||||
|
document_id=document.id,
|
||||||
|
page_number=1,
|
||||||
|
upload_name="page.jpg",
|
||||||
|
filename="page.jpg",
|
||||||
|
file_path="page.jpg",
|
||||||
|
file_hash="a" * 64,
|
||||||
|
file_size_bytes=1,
|
||||||
|
revised_text="human revision",
|
||||||
|
)
|
||||||
|
return await services.sources.create_source(source)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_first_success_is_preferred_and_later_success_remains_candidate(default_session_factory):
|
||||||
|
settings = Settings(openrouter_api_key="test-key", provider_models=None)
|
||||||
|
services = _services(default_session_factory, settings)
|
||||||
|
source = await _seed_source(services)
|
||||||
|
|
||||||
|
first_job = await services.jobs.create_job(Job(document_id=source.document_id))
|
||||||
|
second_job = await services.jobs.create_job(Job(document_id=source.document_id))
|
||||||
|
await services.sources.create_job_source(JobSource(job_id=first_job.id, source_id=source.id))
|
||||||
|
await services.sources.create_job_source(JobSource(job_id=second_job.id, source_id=source.id))
|
||||||
|
|
||||||
|
await services.sources.update_job_source_transcription(
|
||||||
|
job_id=first_job.id,
|
||||||
|
source_id=source.id,
|
||||||
|
text="first result",
|
||||||
|
provider="fixture",
|
||||||
|
model="model-a",
|
||||||
|
)
|
||||||
|
selected = await services.sources.read_source(source.id)
|
||||||
|
first_attempt_id = selected.preferred_execution_attempt_id
|
||||||
|
|
||||||
|
await services.sources.update_job_source_transcription(
|
||||||
|
job_id=second_job.id,
|
||||||
|
source_id=source.id,
|
||||||
|
text="candidate result",
|
||||||
|
provider="fixture",
|
||||||
|
model="model-b",
|
||||||
|
)
|
||||||
|
unchanged = await services.sources.read_source(source.id)
|
||||||
|
attempts = await services.sources.list_execution_attempts(source_id=source.id)
|
||||||
|
|
||||||
|
assert unchanged.raw_transcription == "first result"
|
||||||
|
assert unchanged.preferred_execution_attempt_id == first_attempt_id
|
||||||
|
assert {attempt.raw_transcription for attempt in attempts} == {"first result", "candidate result"}
|
||||||
|
|
||||||
|
candidate = next(attempt for attempt in attempts if attempt.raw_transcription == "candidate result")
|
||||||
|
promoted = await services.sources.promote_machine_attempt(
|
||||||
|
source_id=source.id,
|
||||||
|
execution_attempt_id=candidate.id,
|
||||||
|
)
|
||||||
|
assert promoted.raw_transcription == "candidate result"
|
||||||
|
assert promoted.preferred_execution_attempt_id == candidate.id
|
||||||
|
assert promoted.revised_text == "human revision"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_promotion_rejects_unrelated_attempt(default_session_factory):
|
||||||
|
settings = Settings(openrouter_api_key="test-key", provider_models=None)
|
||||||
|
services = _services(default_session_factory, settings)
|
||||||
|
source = await _seed_source(services)
|
||||||
|
|
||||||
|
with pytest.raises(CandidatePromotionError):
|
||||||
|
await services.sources.promote_machine_attempt(
|
||||||
|
source_id=source.id,
|
||||||
|
execution_attempt_id=uuid4(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_retranscription_job_locks_source_and_frozen_model(default_session_factory, tmp_path):
|
||||||
|
settings = Settings(
|
||||||
|
openrouter_api_key="test-key",
|
||||||
|
prompt_dir=tmp_path,
|
||||||
|
provider_model="vendor/default",
|
||||||
|
provider_models=["vendor/alternate"],
|
||||||
|
)
|
||||||
|
(tmp_path / settings.default_prompt_name).write_text("Transcribe verbatim.", encoding="utf-8")
|
||||||
|
services = _services(default_session_factory, settings)
|
||||||
|
source = await _seed_source(services)
|
||||||
|
|
||||||
|
job = await create_source_retranscription_job(
|
||||||
|
source_id=source.id,
|
||||||
|
model="vendor/alternate",
|
||||||
|
services=services,
|
||||||
|
settings=settings,
|
||||||
|
)
|
||||||
|
loaded = await services.jobs.read_job(job.id)
|
||||||
|
|
||||||
|
assert loaded.purpose == JobPurpose.RETRANSCRIPTION
|
||||||
|
assert loaded.document_id == source.document_id
|
||||||
|
assert loaded.provider == "openrouter"
|
||||||
|
assert loaded.model == "vendor/alternate"
|
||||||
|
assert loaded.user_prompt == "Transcribe verbatim."
|
||||||
|
assert [link.source_id for link in loaded.job_sources] == [source.id]
|
||||||
+26
-1
@@ -12,7 +12,7 @@ from transcription.config import parse_cli_settings
|
|||||||
|
|
||||||
def _make_settings(**overrides) -> Settings:
|
def _make_settings(**overrides) -> Settings:
|
||||||
"""Build a Settings instance with a dummy API key unless overridden."""
|
"""Build a Settings instance with a dummy API key unless overridden."""
|
||||||
defaults = {"openrouter_api_key": "test-key-abc123"}
|
defaults = {"openrouter_api_key": "test-key-abc123", "provider_models": None}
|
||||||
defaults.update(overrides)
|
defaults.update(overrides)
|
||||||
return Settings(**defaults)
|
return Settings(**defaults)
|
||||||
|
|
||||||
@@ -106,6 +106,31 @@ class TestProviderSettings:
|
|||||||
settings = Settings(openrouter_api_key="test-key-abc123")
|
settings = Settings(openrouter_api_key="test-key-abc123")
|
||||||
assert settings.provider_model == "google/gemini-2.5-flash"
|
assert settings.provider_model == "google/gemini-2.5-flash"
|
||||||
|
|
||||||
|
def test_provider_models_defaults_to_default_model(self):
|
||||||
|
settings = _make_settings(provider_model="vendor/default")
|
||||||
|
|
||||||
|
assert settings.provider_models == ("vendor/default",)
|
||||||
|
|
||||||
|
def test_provider_models_are_default_first_trimmed_and_deduplicated(self):
|
||||||
|
settings = _make_settings(
|
||||||
|
provider_model=" vendor/default ",
|
||||||
|
provider_models=["vendor/alternate", "vendor/default", " vendor/other "],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert settings.provider_models == ("vendor/default", "vendor/alternate", "vendor/other")
|
||||||
|
|
||||||
|
def test_provider_models_rejects_empty_list(self):
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
_make_settings(provider_models=[])
|
||||||
|
|
||||||
|
def test_provider_models_loads_json_from_environment(self, monkeypatch):
|
||||||
|
monkeypatch.setenv("PROVIDER_MODEL", "vendor/default")
|
||||||
|
monkeypatch.setenv("PROVIDER_MODELS", '["vendor/alternate","vendor/default"]')
|
||||||
|
|
||||||
|
settings = Settings(openrouter_api_key="test-key-abc123")
|
||||||
|
|
||||||
|
assert settings.provider_models == ("vendor/default", "vendor/alternate")
|
||||||
|
|
||||||
|
|
||||||
class TestPathSettings:
|
class TestPathSettings:
|
||||||
"""Verify filesystem path field types."""
|
"""Verify filesystem path field types."""
|
||||||
|
|||||||
@@ -172,11 +172,23 @@ async def test_v42_upgrade_adds_evidence_tables_without_rewriting_legacy_snapsho
|
|||||||
await upgrade_schema(engine=runtime.engine)
|
await upgrade_schema(engine=runtime.engine)
|
||||||
async with runtime.engine.connect() as connection:
|
async with runtime.engine.connect() as connection:
|
||||||
table_names = set(await connection.run_sync(lambda c: inspect(c).get_table_names()))
|
table_names = set(await connection.run_sync(lambda c: inspect(c).get_table_names()))
|
||||||
|
job_columns = set(
|
||||||
|
await connection.run_sync(
|
||||||
|
lambda c: tuple(column["name"] for column in inspect(c).get_columns("job"))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
source_columns = set(
|
||||||
|
await connection.run_sync(
|
||||||
|
lambda c: tuple(column["name"] for column in inspect(c).get_columns("source"))
|
||||||
|
)
|
||||||
|
)
|
||||||
legacy_snapshot = (
|
legacy_snapshot = (
|
||||||
await connection.execute(text("SELECT raw_api_response FROM job_source WHERE id = 'link-1'"))
|
await connection.execute(text("SELECT raw_api_response FROM job_source WHERE id = 'link-1'"))
|
||||||
).scalar_one()
|
).scalar_one()
|
||||||
|
|
||||||
assert {"execution_attempt", "processing_artifact"}.issubset(table_names)
|
assert {"execution_attempt", "processing_artifact"}.issubset(table_names)
|
||||||
|
assert "purpose" in job_columns
|
||||||
|
assert "preferred_execution_attempt_id" in source_columns
|
||||||
assert "legacy" in legacy_snapshot
|
assert "legacy" in legacy_snapshot
|
||||||
finally:
|
finally:
|
||||||
await dispose_database_runtime()
|
await dispose_database_runtime()
|
||||||
|
|||||||
@@ -50,6 +50,23 @@ class TestPromptArtifact:
|
|||||||
assert "[deleted:" in text
|
assert "[deleted:" in text
|
||||||
assert "[inserted:" in text
|
assert "[inserted:" in text
|
||||||
|
|
||||||
|
def test_prompt_defines_exactly_one_body_medium_marker(self):
|
||||||
|
text = _prompt_text().lower()
|
||||||
|
for marker in (
|
||||||
|
"[document body handwritten]",
|
||||||
|
"[document body typewritten]",
|
||||||
|
"[document body typeset]",
|
||||||
|
"[document body mixed]",
|
||||||
|
):
|
||||||
|
assert marker in text
|
||||||
|
assert "exactly one" in text
|
||||||
|
assert "typewriter defects are not handwriting" in text
|
||||||
|
|
||||||
|
def test_prompt_preserves_structured_layout_associations(self):
|
||||||
|
text = _prompt_text().lower()
|
||||||
|
for phrase in ("tables of contents", "dotted-leader", "page-reference", "tables and forms", "columns"):
|
||||||
|
assert phrase in text
|
||||||
|
|
||||||
|
|
||||||
class TestPromptConfiguration:
|
class TestPromptConfiguration:
|
||||||
def test_builds_validated_immutable_prompt_provenance(self, tmp_path):
|
def test_builds_validated_immutable_prompt_provenance(self, tmp_path):
|
||||||
|
|||||||
Reference in New Issue
Block a user