generated from john/python-template
V4.7 Phase 1: ingest orientation normalization, ProcessingArtifact removal
Move orientation normalization to the Source-ingest boundary and delete the ProcessingArtifact subsystem it was built to serve. Stored pages are now already upright, so nothing downstream derives a rotated copy: every stored byte is the byte a provider is later sent. Rotation runs in store_source_file ahead of hashing, so source.file_hash and file_size_bytes describe exactly what is on disk. normalize_orientation becomes bytes-in / bytes-out, and JPEG output reuses the source quantization tables and chroma subsampling instead of re-quantizing at a fixed quality - measured at 50.3-56.1 dB PSNR at -6% size, against 50.0-53.5 dB at +38% for quality=95. ProcessingArtifact held 2 rows against 77 successful transcriptions; the subsystem effectively never ran. Deleting it removes the artifact cluster from sources.py, the derivative resolution in workflows.py, the pre-provider commit that only existed to make an artifact row durable, and the artifact evidence dump from the Source detail page. The transcription_quality_warnings payload folds into execution_attempt.normalized_metadata, so that feature keeps working without the table. tools/migrate_v46_to_v47.py carries steps 1 and 2: it rotated the 58 stored images carrying EXIF orientation 3 in place, updated their recorded hash and size, dropped processing_artifact and removed its one external file. It is idempotent, keyed on state rather than a version marker. tools/migrate_v45_to_v46.py is deleted. That migration is complete, and after V4.7 it would restore a V4.5 backup into a schema that no longer matches. Also fixes tests/test_config.py, which read the developer's local .env and failed whenever WORKER_MAX_RETRIES was set. Co-authored-by: Copilot App <[email protected]>
This commit is contained in:
zoltan57
Copilot App
@@ -19,7 +19,6 @@ from transcription.db.models import Document
|
||||
from transcription.db.models import Job
|
||||
from transcription.db.models import JobSource
|
||||
from transcription.db.models import JobSourceStatus
|
||||
from transcription.db.models import ProcessingArtifact
|
||||
from transcription.db.models import Source
|
||||
from transcription.providers.base import ProviderError
|
||||
from transcription.providers.base import TranscriptionResult
|
||||
@@ -28,9 +27,7 @@ from transcription.providers.openrouter import OpenRouterTranscriptionProvider
|
||||
from transcription.services.documents import DocumentService
|
||||
from transcription.services.jobs import JobDeleteBlockedError
|
||||
from transcription.services.jobs import JobService
|
||||
from transcription.services.sources import SourceDeleteBlockedError
|
||||
from transcription.services.sources import SourceService
|
||||
from transcription.services.sources import TranscriptionError
|
||||
from transcription.services.sources import transcribe_document_image
|
||||
|
||||
|
||||
@@ -249,34 +246,9 @@ async def test_attempts_are_append_only_and_exported_with_integrity(default_sess
|
||||
assert attempts[1].status == JobSourceStatus.TRANSCRIBED
|
||||
assert attempts[1].raw_transcription == "second succeeded"
|
||||
|
||||
payload = {"words": [{"text": "second", "polygon": [0, 0, 1, 1]}]}
|
||||
payload_bytes = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
|
||||
artifact = await sources.create_processing_artifact(
|
||||
ProcessingArtifact(
|
||||
source_id=source.id,
|
||||
execution_attempt_id=attempts[1].id,
|
||||
artifact_type="ocr.words",
|
||||
media_type="application/json",
|
||||
schema_name="example.ocr.words",
|
||||
schema_version="1",
|
||||
producer="fixture",
|
||||
producer_version="1",
|
||||
inline_payload=payload,
|
||||
payload_sha256=hashlib.sha256(payload_bytes).hexdigest(),
|
||||
byte_size=len(payload_bytes),
|
||||
coordinate_metadata={
|
||||
"units": "normalized",
|
||||
"origin": "top-left",
|
||||
"width": 1,
|
||||
"height": 1,
|
||||
"transformations": [],
|
||||
},
|
||||
)
|
||||
)
|
||||
export = await sources.build_evidence_export(source_id=source.id)
|
||||
assert _json_object(export["source"])["digest_sha256"] == "a" * 64
|
||||
assert [_json_object(item)["attempt_number"] for item in _json_array(export["attempts"])] == [1, 2]
|
||||
assert _json_object(_json_array(export["artifacts"])[0])["id"] == str(artifact.id)
|
||||
assert "file_path" not in json.dumps(export)
|
||||
with pytest.raises(JobDeleteBlockedError):
|
||||
await jobs.delete_job_with_guardrails(job_id=job.id)
|
||||
@@ -285,7 +257,6 @@ async def test_attempts_are_append_only_and_exported_with_integrity(default_sess
|
||||
latest_job_source = detail.latest_job_source
|
||||
assert latest_job_source is not None
|
||||
assert latest_job_source.execution_attempts == []
|
||||
assert detail.processing_artifacts == []
|
||||
latest_attempt = await sources.read_latest_execution_attempt(job_source_id=latest_job_source.id)
|
||||
assert latest_attempt is not None
|
||||
assert latest_attempt.attempt.attempt_number == 2
|
||||
@@ -304,89 +275,6 @@ def test_benchmark_scoring_preserves_literal_differences():
|
||||
assert score.assessment.silent_normalizations == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_large_json_artifact_uses_constrained_atomic_storage(
|
||||
default_session_factory,
|
||||
tmp_path,
|
||||
):
|
||||
settings = Settings(
|
||||
openrouter_api_key="test-key",
|
||||
artifact_dir=tmp_path / "artifacts",
|
||||
artifact_inline_threshold_bytes=10,
|
||||
)
|
||||
documents = DocumentService(session_factory=default_session_factory, settings=settings)
|
||||
sources = SourceService(session_factory=default_session_factory, settings=settings)
|
||||
document = await documents.create_document(Document(name="External Artifact"))
|
||||
source = await sources.create_source(
|
||||
Source(
|
||||
document_id=document.id,
|
||||
page_number=1,
|
||||
upload_name="page.png",
|
||||
filename="page.png",
|
||||
file_path="page.png",
|
||||
file_hash="b" * 64,
|
||||
file_size_bytes=10,
|
||||
)
|
||||
)
|
||||
|
||||
artifact = await sources.create_json_artifact(
|
||||
source_id=source.id,
|
||||
execution_attempt_id=None,
|
||||
artifact_type="ocr.layout",
|
||||
schema_name="example.layout",
|
||||
schema_version="1",
|
||||
producer="fixture",
|
||||
producer_version="1",
|
||||
payload={"blocks": [{"text": "long enough to be external"}]},
|
||||
)
|
||||
|
||||
assert artifact.inline_payload is None
|
||||
assert artifact.external_reference is not None
|
||||
stored_path = settings.artifact_dir / artifact.external_reference
|
||||
assert stored_path.is_file()
|
||||
assert hashlib.sha256(stored_path.read_bytes()).hexdigest() == artifact.payload_sha256
|
||||
with pytest.raises(SourceDeleteBlockedError):
|
||||
await sources.delete_unlinked_source(source_id=source.id)
|
||||
|
||||
stored_path.write_bytes(b'{"tampered":true}')
|
||||
with pytest.raises(TranscriptionError, match="integrity verification"):
|
||||
await sources.build_evidence_export(source_id=source.id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_inline_artifact_with_incorrect_integrity(default_session_factory):
|
||||
documents = DocumentService(session_factory=default_session_factory)
|
||||
sources = SourceService(session_factory=default_session_factory)
|
||||
document = await documents.create_document(Document(name="Inline Integrity"))
|
||||
source = await sources.create_source(
|
||||
Source(
|
||||
document_id=document.id,
|
||||
page_number=1,
|
||||
upload_name="page.png",
|
||||
filename="page.png",
|
||||
file_path="page.png",
|
||||
file_hash="d" * 64,
|
||||
file_size_bytes=10,
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(TranscriptionError, match="integrity verification"):
|
||||
await sources.create_processing_artifact(
|
||||
ProcessingArtifact(
|
||||
source_id=source.id,
|
||||
artifact_type="ocr.words",
|
||||
media_type="application/json",
|
||||
schema_name="example.words",
|
||||
schema_version="1",
|
||||
producer="fixture",
|
||||
producer_version="1",
|
||||
inline_payload={"words": []},
|
||||
payload_sha256="0" * 64,
|
||||
byte_size=1,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_standalone_transcription_closes_locally_created_provider(tmp_path, monkeypatch):
|
||||
image_path = tmp_path / "page.png"
|
||||
|
||||
Reference in New Issue
Block a user