generated from john/python-template
zoltan57andCopilot App
f86c0ff27b
Move orientation normalization to the Source-ingest boundary and delete the ProcessingArtifact subsystem it was built to serve. Stored pages are now already upright, so nothing downstream derives a rotated copy: every stored byte is the byte a provider is later sent. Rotation runs in store_source_file ahead of hashing, so source.file_hash and file_size_bytes describe exactly what is on disk. normalize_orientation becomes bytes-in / bytes-out, and JPEG output reuses the source quantization tables and chroma subsampling instead of re-quantizing at a fixed quality - measured at 50.3-56.1 dB PSNR at -6% size, against 50.0-53.5 dB at +38% for quality=95. ProcessingArtifact held 2 rows against 77 successful transcriptions; the subsystem effectively never ran. Deleting it removes the artifact cluster from sources.py, the derivative resolution in workflows.py, the pre-provider commit that only existed to make an artifact row durable, and the artifact evidence dump from the Source detail page. The transcription_quality_warnings payload folds into execution_attempt.normalized_metadata, so that feature keeps working without the table. tools/migrate_v46_to_v47.py carries steps 1 and 2: it rotated the 58 stored images carrying EXIF orientation 3 in place, updated their recorded hash and size, dropped processing_artifact and removed its one external file. It is idempotent, keyed on state rather than a version marker. tools/migrate_v45_to_v46.py is deleted. That migration is complete, and after V4.7 it would restore a V4.5 backup into a schema that no longer matches. Also fixes tests/test_config.py, which read the developer's local .env and failed whenever WORKER_MAX_RETRIES was set. Co-authored-by: Copilot App <[email protected]>
160 lines
4.9 KiB
Python
160 lines
4.9 KiB
Python
"""Tests for ingest-time orientation normalization."""
|
|
|
|
import hashlib
|
|
import io
|
|
|
|
import pytest
|
|
from PIL import Image
|
|
from PIL import JpegImagePlugin
|
|
|
|
from transcription.config import Settings
|
|
from transcription.services.normalization import normalize_orientation
|
|
from transcription.services.normalization import normalize_orientation_async
|
|
from transcription.services.store import store_source_file
|
|
|
|
|
|
def _oriented_jpeg(orientation: int) -> bytes:
|
|
image = Image.new("RGB", (2, 3))
|
|
image.putdata(
|
|
[
|
|
(255, 0, 0),
|
|
(255, 0, 0),
|
|
(0, 255, 0),
|
|
(0, 255, 0),
|
|
(0, 0, 255),
|
|
(0, 0, 255),
|
|
]
|
|
)
|
|
exif = Image.Exif()
|
|
exif[274] = orientation
|
|
buffer = io.BytesIO()
|
|
image.save(buffer, format="JPEG", quality=100, subsampling=0, exif=exif)
|
|
return buffer.getvalue()
|
|
|
|
|
|
def _oriented_image(orientation: int, image_format: str) -> bytes:
|
|
image = Image.new("RGB", (2, 3), color="white")
|
|
exif = Image.Exif()
|
|
exif[274] = orientation
|
|
buffer = io.BytesIO()
|
|
image.save(buffer, format=image_format, exif=exif)
|
|
return buffer.getvalue()
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_orientation_three_is_physically_rotated_and_metadata_removed():
|
|
result = normalize_orientation(_oriented_jpeg(3), media_type="image/jpeg")
|
|
|
|
assert result is not None
|
|
assert result.original_orientation == 3
|
|
assert result.applied_rotation_degrees == 180
|
|
with Image.open(io.BytesIO(result.content)) as normalized:
|
|
assert normalized.getexif().get(274, 1) == 1
|
|
pixel = normalized.getpixel((0, 0))
|
|
assert isinstance(pixel, tuple)
|
|
assert pixel[2] > pixel[0]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_orientation_one_is_noop():
|
|
assert normalize_orientation(_oriented_jpeg(1), media_type="image/jpeg") is None
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_normalization_is_idempotent():
|
|
once = normalize_orientation(_oriented_jpeg(3), media_type="image/jpeg")
|
|
|
|
assert once is not None
|
|
assert normalize_orientation(once.content, media_type="image/jpeg") is None
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_jpeg_reencode_reuses_source_quantization_tables():
|
|
"""Reusing the source tables is what keeps the rewrite small and near-lossless."""
|
|
original = _oriented_jpeg(3)
|
|
result = normalize_orientation(original, media_type="image/jpeg")
|
|
|
|
assert result is not None
|
|
with (
|
|
Image.open(io.BytesIO(original)) as before,
|
|
Image.open(io.BytesIO(result.content)) as after,
|
|
):
|
|
assert isinstance(before, JpegImagePlugin.JpegImageFile)
|
|
assert isinstance(after, JpegImagePlugin.JpegImageFile)
|
|
assert after.quantization == before.quantization
|
|
|
|
|
|
@pytest.mark.unit
|
|
@pytest.mark.parametrize(
|
|
("image_format", "media_type"),
|
|
[("PNG", "image/png"), ("TIFF", "image/tiff")],
|
|
)
|
|
def test_supported_non_jpeg_orientation_is_normalized(image_format, media_type):
|
|
result = normalize_orientation(_oriented_image(6, image_format), media_type=media_type)
|
|
|
|
assert result is not None
|
|
assert result.applied_rotation_degrees == 90
|
|
with Image.open(io.BytesIO(result.content)) as normalized:
|
|
assert normalized.getexif().get(274, 1) == 1
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_unsupported_media_type_is_left_alone():
|
|
assert normalize_orientation(b"not-an-image", media_type="application/pdf") is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_async_wrapper_matches_sync_result():
|
|
"""[MED-01]: Pillow work runs off the event loop."""
|
|
original = _oriented_jpeg(3)
|
|
|
|
result = await normalize_orientation_async(original, media_type="image/jpeg")
|
|
expected = normalize_orientation(original, media_type="image/jpeg")
|
|
|
|
assert result is not None
|
|
assert expected is not None
|
|
assert result.content == expected.content
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("orientation", [3, 6, 8])
|
|
async def test_stored_source_never_retains_exif_orientation(tmp_path, orientation):
|
|
settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
upload_dir=tmp_path / "uploads",
|
|
provider_models=None,
|
|
)
|
|
|
|
stored = await store_source_file(
|
|
filename="page.jpg",
|
|
file_bytes=_oriented_jpeg(orientation),
|
|
settings=settings,
|
|
)
|
|
|
|
stored_bytes = stored.path.read_bytes()
|
|
with Image.open(io.BytesIO(stored_bytes)) as image:
|
|
assert image.getexif().get(274, 1) == 1
|
|
assert stored.file_hash == hashlib.sha256(stored_bytes).hexdigest()
|
|
assert stored.file_size_bytes == len(stored_bytes)
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
async def test_upright_source_is_stored_byte_for_byte(tmp_path):
|
|
settings = Settings(
|
|
openrouter_api_key="test-key",
|
|
upload_dir=tmp_path / "uploads",
|
|
provider_models=None,
|
|
)
|
|
original = _oriented_jpeg(1)
|
|
|
|
stored = await store_source_file(
|
|
filename="page.jpg",
|
|
file_bytes=original,
|
|
settings=settings,
|
|
)
|
|
|
|
assert stored.path.read_bytes() == original
|
|
assert stored.file_hash == hashlib.sha256(original).hexdigest()
|