V4.11 Added tags + lots of little changes to the UI

tests/test_storage_reconciliation.py

@@ -0,0 +1,116 @@
+"""Integrity checks for document/source filesystem-to-database reconciliation."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from uuid import UUID
+from uuid import uuid4
+
+import pytest
+from sqlmodel import func
+from sqlmodel import select
+
+from transcription.config import Settings
+from transcription.db import session_scope
+from transcription.db.models import Document
+from transcription.db.models import Source
+
+
+def _document_folder_ids(root: Path) -> set[str]:
+    documents_root = root / "documents"
+    if not documents_root.exists():
+        return set()
+    return {entry.name for entry in documents_root.iterdir() if entry.is_dir()}
+
+
+async def _document_ids(settings: Settings) -> set[str]:
+    async with session_scope(settings=settings) as session:
+        rows = await session.exec(select(Document.id))
+        return {str(item) for item in rows.all()}
+
+
+async def _source_counts_by_document(settings: Settings) -> dict[str, int]:
+    async with session_scope(settings=settings) as session:
+        rows = await session.exec(
+            select(Source.document_id, func.count(Source.id)).group_by(Source.document_id)
+        )
+        return {str(document_id): int(count) for document_id, count in rows}
+
+
+def _source_file_count_for_document(root: Path, document_id: str) -> int:
+    directory = root / "documents" / document_id
+    if not directory.exists():
+        return 0
+    return sum(1 for entry in directory.iterdir() if entry.is_file())
+
+
+async def assert_storage_reconciliation(*, upload_dir: Path, settings: Settings) -> None:
+    folder_ids = _document_folder_ids(upload_dir)
+    doc_ids = await _document_ids(settings)
+
+    missing_in_table = sorted(folder_ids - doc_ids)
+    missing_in_folders = sorted(doc_ids - folder_ids)
+    if missing_in_table:
+        folder = missing_in_table[0]
+        raise AssertionError(
+            "Document directory count and document.doc_id count do not agree. "
+            f"./data/documents/{folder} does not appear in document table"
+        )
+    if missing_in_folders:
+        doc_id = missing_in_folders[0]
+        raise AssertionError(
+            "Document directory count and document.doc_id count do not agree. "
+            f"document.doc_id {doc_id} has no corresponding folder in ./data/documents"
+        )
+
+    source_counts = await _source_counts_by_document(settings)
+    for document_id in sorted(doc_ids):
+        db_count = source_counts.get(document_id, 0)
+        file_count = _source_file_count_for_document(upload_dir, document_id)
+        if file_count > db_count:
+            raise AssertionError(
+                "Source file count and source.source_id count do not agree. "
+                f"[UPLOAD_DIR]/documents/{document_id}/ contains file(s) with no source row"
+            )
+        if db_count > file_count:
+            raise AssertionError(
+                "Source file count and source.source_id count do not agree. "
+                f"source.source_id rows exist without files in [UPLOAD_DIR]/documents/{document_id}"
+            )
+
+
+@pytest.mark.asyncio
+async def test_storage_reconciliation_passes_for_matching_counts(tmp_path, default_settings: Settings):
+    upload_dir = tmp_path / "uploads"
+    document_id = str(uuid4())
+    document_uuid = UUID(document_id)
+    file_name = f"{uuid4()}.png"
+    (upload_dir / "documents" / document_id).mkdir(parents=True, exist_ok=True)
+    (upload_dir / "documents" / document_id / file_name).write_bytes(b"ok")
+
+    async with session_scope(settings=default_settings) as session:
+        session.add(Document(id=document_uuid, name="Doc"))
+        session.add(
+            Source(
+                document_id=document_uuid,
+                page_number=1,
+                upload_name=file_name,
+                filename=file_name,
+                file_path=f"documents/{document_id}/{file_name}",
+                file_hash="a" * 64,
+                file_size_bytes=2,
+            )
+        )
+        await session.commit()
+
+    await assert_storage_reconciliation(upload_dir=upload_dir, settings=default_settings)
+
+
+@pytest.mark.asyncio
+async def test_storage_reconciliation_reports_actionable_mismatch_message(tmp_path, default_settings: Settings):
+    upload_dir = tmp_path / "uploads"
+    orphan_dir = upload_dir / "documents" / str(uuid4())
+    orphan_dir.mkdir(parents=True, exist_ok=True)
+
+    with pytest.raises(AssertionError, match="document\\.doc_id count do not agree"):
+        await assert_storage_reconciliation(upload_dir=upload_dir, settings=default_settings)