V5.1 Update tests and documentation

tests/test_storage_reconciliation.py

@@ -16,6 +16,10 @@ from transcription.db.models import Document
 from transcription.db.models import Source
 
 
+def _normalize_identifier(value: str) -> str:
+    return value.replace("-", "").strip().lower()
+
+
 def _document_folder_ids(root: Path) -> set[str]:
     documents_root = root / "documents"
     if not documents_root.exists():
@@ -47,37 +51,53 @@ def _source_file_count_for_document(root: Path, document_id: str) -> int:
 async def assert_storage_reconciliation(*, upload_dir: Path, settings: Settings) -> None:
     folder_ids = _document_folder_ids(upload_dir)
     doc_ids = await _document_ids(settings)
-
-    missing_in_table = sorted(folder_ids - doc_ids)
-    missing_in_folders = sorted(doc_ids - folder_ids)
-    if missing_in_table:
-        folder = missing_in_table[0]
-        raise AssertionError(
-            "Document directory count and document.doc_id count do not agree. "
-            f"./data/documents/{folder} does not appear in document table"
-        )
-    if missing_in_folders:
-        doc_id = missing_in_folders[0]
-        raise AssertionError(
-            "Document directory count and document.doc_id count do not agree. "
-            f"document.doc_id {doc_id} has no corresponding folder in ./data/documents"
-        )
-
+    folder_by_normalized = {_normalize_identifier(folder_id): folder_id for folder_id in folder_ids}
+    doc_by_normalized = {_normalize_identifier(doc_id): doc_id for doc_id in doc_ids}
     source_counts = await _source_counts_by_document(settings)
-    for document_id in sorted(doc_ids):
-        db_count = source_counts.get(document_id, 0)
-        file_count = _source_file_count_for_document(upload_dir, document_id)
-        if file_count > db_count:
-            raise AssertionError(
-                "Source file count and source.source_id count do not agree. "
-                f"[UPLOAD_DIR]/documents/{document_id}/ contains file(s) with no source row"
+    source_counts_by_normalized = {
+        _normalize_identifier(document_id): count for document_id, count in source_counts.items()
+    }
+    mismatches: list[str] = []
+
+    missing_in_table = sorted(set(folder_by_normalized) - set(doc_by_normalized))
+    for folder_key in missing_in_table:
+        folder_name = folder_by_normalized[folder_key]
+        mismatches.append(
+            f"document-folder-without-row: ./data/documents/{folder_name} has no matching document.doc_id"
+        )
+
+    missing_in_folders = sorted(set(doc_by_normalized) - set(folder_by_normalized))
+    for doc_key in missing_in_folders:
+        doc_id = doc_by_normalized[doc_key]
+        source_count = source_counts_by_normalized.get(doc_key, 0)
+        mismatches.append(
+            "document-row-without-folder: "
+            f"document.doc_id {doc_id} has no corresponding folder in ./data/documents "
+            f"(source rows: {source_count})"
+        )
+
+    for doc_key in sorted(doc_by_normalized):
+        doc_id = doc_by_normalized[doc_key]
+        folder_name = folder_by_normalized.get(doc_key)
+        db_count = source_counts_by_normalized.get(doc_key, 0)
+        file_count = _source_file_count_for_document(upload_dir, folder_name) if folder_name is not None else 0
+        if db_count != file_count:
+            folder_display = (
+                f"./data/documents/{folder_name}" if folder_name is not None else "./data/documents/<missing-folder>"
             )
-        if db_count > file_count:
-            raise AssertionError(
-                "Source file count and source.source_id count do not agree. "
-                f"source.source_id rows exist without files in [UPLOAD_DIR]/documents/{document_id}"
+            mismatches.append(
+                "source-count-mismatch: "
+                f"document.doc_id {doc_id} -> source rows: {db_count}, files in {folder_display}: {file_count}"
             )
 
+    if mismatches:
+        report = "\n".join(f"- {item}" for item in mismatches)
+        raise AssertionError(
+            "Storage reconciliation mismatch(es) detected.\n"
+            f"Reconciling item count: {len(mismatches)}\n"
+            f"{report}"
+        )
+
 
 @pytest.mark.asyncio
 async def test_storage_reconciliation_passes_for_matching_counts(tmp_path, default_settings: Settings):
@@ -112,5 +132,20 @@ async def test_storage_reconciliation_reports_actionable_mismatch_message(tmp_pa
     orphan_dir = upload_dir / "documents" / str(uuid4())
     orphan_dir.mkdir(parents=True, exist_ok=True)
 
-    with pytest.raises(AssertionError, match="document\\.doc_id count do not agree"):
+    with pytest.raises(AssertionError, match="Reconciling item count:"):
+        await assert_storage_reconciliation(upload_dir=upload_dir, settings=default_settings)
+
+
+@pytest.mark.asyncio
+async def test_storage_reconciliation_reports_missing_document_folder_even_without_sources(
+    tmp_path, default_settings: Settings
+):
+    upload_dir = tmp_path / "uploads"
+    missing_folder_doc_id = uuid4()
+
+    async with session_scope(settings=default_settings) as session:
+        session.add(Document(id=missing_folder_doc_id, name="Needs folder"))
+        await session.commit()
+
+    with pytest.raises(AssertionError, match="document-row-without-folder:"):
         await assert_storage_reconciliation(upload_dir=upload_dir, settings=default_settings)