V4.11 Added tags + lots of little changes to the UI

tests/services/test_document_service.py

@@ -5,13 +5,16 @@ from datetime import datetime
 from uuid import uuid4
 
 import pytest
+from sqlmodel import select
 
 from transcription.config import Settings
 from transcription.db.models import Document
 from transcription.db.models import DocumentPerson
+from transcription.db.models import DocumentTag
 from transcription.db.models import Job
 from transcription.db.models import Person
 from transcription.db.models import Source
+from transcription.db.models import Tag
 from transcription.services.documents import DocumentDeleteBlockedError
 from transcription.services.documents import DocumentError
 from transcription.services.documents import DocumentService
@@ -311,3 +314,26 @@ async def test_update_document_person_changes_role_id(default_session_factory):
     )
 
     assert updated.role_id == recipient_role.id
+
+
+@pytest.mark.asyncio
+async def test_sync_document_tags_by_labels_creates_and_replaces_tags(default_session_factory):
+    documents = DocumentService(session_factory=default_session_factory)
+    document = await documents.create_document(Document(id=uuid4(), name="tagged-doc"))
+
+    await documents.sync_document_tags_by_labels(document_id=document.id, labels=["Family", "Census"])
+    await documents.sync_document_tags_by_labels(document_id=document.id, labels=["Census", "Research"])
+
+    async with documents._session_scope() as session:
+        links = (await session.exec(select(DocumentTag).where(DocumentTag.document_id == document.id))).all()
+        tags = (await session.exec(select(Tag))).all()
+
+    assert len(links) == 2
+    linked_ids = {link.tag_id for link in links}
+    linked_labels = {tag.label for tag in tags if tag.id in linked_ids}
+    assert linked_labels == {"Census", "Research"}
+
+    listed = await documents.list_documents()
+    assert len(listed) == 1
+    listed_labels = {link.tag_ref.label for link in listed[0].document_tags if link.tag_ref is not None}
+    assert listed_labels == {"Census", "Research"}

tests/services/test_v44_workflows.py

@@ -31,6 +31,7 @@ async def test_create_document_with_people_rolls_back_on_invalid_person(default_
         await create_document_with_people(
             document=Document(name="Must roll back"),
             links=[DocumentPersonInput(person_id=uuid4(), role_id=role.id)],
+            tag_labels=[],
             documents=documents,
             people=people,
         )
@@ -48,6 +49,7 @@ async def test_update_document_with_people_rolls_back_document_and_links(default
     document = await create_document_with_people(
         document=Document(name="Original name"),
         links=[DocumentPersonInput(person_id=person.id, role_id=role.id)],
+        tag_labels=[],
         documents=documents,
         people=people,
     )
@@ -63,6 +65,7 @@ async def test_update_document_with_people_rolls_back_document_and_links(default
         await update_document_with_people(
             document=candidate,
             links=[DocumentPersonInput(person_id=person.id, role_id=inactive.id)],
+            tag_labels=[],
             documents=documents,
             people=people,
         )

tests/test_db.py

@@ -41,9 +41,11 @@ async def test_create_all_creates_expected_tables(tmp_path):
 
         assert "document" in table_names
         assert "document_type" in table_names
+        assert "tag" in table_names
         assert "person" in table_names
         assert "person_role" in table_names
         assert "document_person" in table_names
+        assert "document_tag" in table_names
         assert "job" in table_names
         assert "source" in table_names
         assert "job_source" in table_names
@@ -130,15 +132,19 @@ async def test_create_all_declares_hot_path_indexes(tmp_path):
                 database = inspect(sync_connection)
                 indexes = {
                     table: [index["column_names"] for index in database.get_indexes(table)]
-                    for table in ("job", "source", "job_source", "document", "document_person")
+                    for table in ("job", "source", "job_source", "document", "document_person", "document_tag")
                 }
                 job_source_unique = [
                     constraint["column_names"]
                     for constraint in database.get_unique_constraints("job_source")
                 ]
-                return indexes, job_source_unique
+                document_tag_unique = [
+                    constraint["column_names"]
+                    for constraint in database.get_unique_constraints("document_tag")
+                ]
+                return indexes, job_source_unique, document_tag_unique
 
-            indexes, job_source_unique = await connection.run_sync(collect)
+            indexes, job_source_unique, document_tag_unique = await connection.run_sync(collect)
 
         assert ["status", "date_created"] in indexes["job"]
         assert ["document_id"] in indexes["job"]
@@ -150,6 +156,9 @@ async def test_create_all_declares_hot_path_indexes(tmp_path):
         assert ["document_type_id"] in indexes["document"]
         for column in ("document_id", "person_id", "role_id"):
             assert [column] in indexes["document_person"]
+        assert ["document_id"] in indexes["document_tag"]
+        assert ["tag_id"] in indexes["document_tag"]
+        assert ["document_id", "tag_id"] in document_tag_unique
     finally:
         await dispose_database_runtime()
 

tests/test_storage_reconciliation.py

@@ -0,0 +1,116 @@
+"""Integrity checks for document/source filesystem-to-database reconciliation."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from uuid import UUID
+from uuid import uuid4
+
+import pytest
+from sqlmodel import func
+from sqlmodel import select
+
+from transcription.config import Settings
+from transcription.db import session_scope
+from transcription.db.models import Document
+from transcription.db.models import Source
+
+
+def _document_folder_ids(root: Path) -> set[str]:
+    documents_root = root / "documents"
+    if not documents_root.exists():
+        return set()
+    return {entry.name for entry in documents_root.iterdir() if entry.is_dir()}
+
+
+async def _document_ids(settings: Settings) -> set[str]:
+    async with session_scope(settings=settings) as session:
+        rows = await session.exec(select(Document.id))
+        return {str(item) for item in rows.all()}
+
+
+async def _source_counts_by_document(settings: Settings) -> dict[str, int]:
+    async with session_scope(settings=settings) as session:
+        rows = await session.exec(
+            select(Source.document_id, func.count(Source.id)).group_by(Source.document_id)
+        )
+        return {str(document_id): int(count) for document_id, count in rows}
+
+
+def _source_file_count_for_document(root: Path, document_id: str) -> int:
+    directory = root / "documents" / document_id
+    if not directory.exists():
+        return 0
+    return sum(1 for entry in directory.iterdir() if entry.is_file())
+
+
+async def assert_storage_reconciliation(*, upload_dir: Path, settings: Settings) -> None:
+    folder_ids = _document_folder_ids(upload_dir)
+    doc_ids = await _document_ids(settings)
+
+    missing_in_table = sorted(folder_ids - doc_ids)
+    missing_in_folders = sorted(doc_ids - folder_ids)
+    if missing_in_table:
+        folder = missing_in_table[0]
+        raise AssertionError(
+            "Document directory count and document.doc_id count do not agree. "
+            f"./data/documents/{folder} does not appear in document table"
+        )
+    if missing_in_folders:
+        doc_id = missing_in_folders[0]
+        raise AssertionError(
+            "Document directory count and document.doc_id count do not agree. "
+            f"document.doc_id {doc_id} has no corresponding folder in ./data/documents"
+        )
+
+    source_counts = await _source_counts_by_document(settings)
+    for document_id in sorted(doc_ids):
+        db_count = source_counts.get(document_id, 0)
+        file_count = _source_file_count_for_document(upload_dir, document_id)
+        if file_count > db_count:
+            raise AssertionError(
+                "Source file count and source.source_id count do not agree. "
+                f"[UPLOAD_DIR]/documents/{document_id}/ contains file(s) with no source row"
+            )
+        if db_count > file_count:
+            raise AssertionError(
+                "Source file count and source.source_id count do not agree. "
+                f"source.source_id rows exist without files in [UPLOAD_DIR]/documents/{document_id}"
+            )
+
+
+@pytest.mark.asyncio
+async def test_storage_reconciliation_passes_for_matching_counts(tmp_path, default_settings: Settings):
+    upload_dir = tmp_path / "uploads"
+    document_id = str(uuid4())
+    document_uuid = UUID(document_id)
+    file_name = f"{uuid4()}.png"
+    (upload_dir / "documents" / document_id).mkdir(parents=True, exist_ok=True)
+    (upload_dir / "documents" / document_id / file_name).write_bytes(b"ok")
+
+    async with session_scope(settings=default_settings) as session:
+        session.add(Document(id=document_uuid, name="Doc"))
+        session.add(
+            Source(
+                document_id=document_uuid,
+                page_number=1,
+                upload_name=file_name,
+                filename=file_name,
+                file_path=f"documents/{document_id}/{file_name}",
+                file_hash="a" * 64,
+                file_size_bytes=2,
+            )
+        )
+        await session.commit()
+
+    await assert_storage_reconciliation(upload_dir=upload_dir, settings=default_settings)
+
+
+@pytest.mark.asyncio
+async def test_storage_reconciliation_reports_actionable_mismatch_message(tmp_path, default_settings: Settings):
+    upload_dir = tmp_path / "uploads"
+    orphan_dir = upload_dir / "documents" / str(uuid4())
+    orphan_dir.mkdir(parents=True, exist_ok=True)
+
+    with pytest.raises(AssertionError, match="document\\.doc_id count do not agree"):
+        await assert_storage_reconciliation(upload_dir=upload_dir, settings=default_settings)

tests/ui/conftest.py

@@ -23,6 +23,7 @@ from transcription.db import session as db_session_module
 from transcription.db import session_scope
 from transcription.db.models import Document
 from transcription.db.models import DocumentPerson
+from transcription.db.models import DocumentTag
 from transcription.db.models import ExecutionAttempt
 from transcription.db.models import Job
 from transcription.db.models import JobSource
@@ -30,6 +31,7 @@ from transcription.db.models import JobSourceStatus
 from transcription.db.models import JobStatus
 from transcription.db.models import Person
 from transcription.db.models import Source
+from transcription.db.models import Tag
 
 
 @pytest.fixture(scope="session")
@@ -73,11 +75,13 @@ async def clear_ui_database(
     )
     async with session_scope(session_factory=app.state.runtime.session_factory) as session:
         await session.exec(delete(JobSource))
+        await session.exec(delete(DocumentTag))
         await session.exec(delete(DocumentPerson))
         await session.exec(delete(Source))
         await session.exec(delete(Job))
         await session.exec(delete(Document))
         await session.exec(delete(Person))
+        await session.exec(delete(Tag))
         await session.commit()
 
 

tests/ui/test_documents_page.py

@@ -14,6 +14,7 @@ from transcription.db.models import Job
 from transcription.db.models import Person
 from transcription.db.models import PersonRole
 from transcription.db.models import Source
+from transcription.ui.pages.documents_page import _resolve_selected_tag_labels
 
 # --- Helper Fixtures ---
 
@@ -81,9 +82,10 @@ class TestDocumentsPageRendering:
         assert response.status_code == 200
         assert "1924 Postcard" in response.text
         assert "Postcard" in response.text
-        assert "PC-001" in response.text
         assert "Document Date" in response.text
         assert "Author" in response.text
+        assert "# Sources" in response.text
+        assert "Archive Ref" not in response.text
 
     def test_document_create_page_renders_form(self, app_client):
         _, client = app_client
@@ -95,6 +97,7 @@ class TestDocumentsPageRendering:
         assert "Document name" in response.text
         assert "Linked People" in response.text
         assert "Document type" in response.text
+        assert "Tags" in response.text
 
     @pytest.mark.asyncio
     async def test_document_create_page_preselects_person_with_disambiguating_label(self, app_client):
@@ -209,3 +212,10 @@ class TestDocumentsPageRendering:
         assert "Delete Document" in response.text
         assert "Delete document permanently" in response.text
         assert "Delete is blocked" not in response.text
+
+
+def test_resolve_selected_tag_labels_handles_multiple_payload_shapes():
+    assert _resolve_selected_tag_labels("Family") == ["Family"]
+    assert _resolve_selected_tag_labels(["Family", "Research"]) == ["Family", "Research"]
+    assert _resolve_selected_tag_labels([{"label": "Family"}, {"value": "Research"}]) == ["Family", "Research"]
+    assert set(_resolve_selected_tag_labels({"value": {"Family", "Research"}})) == {"Family", "Research"}

tests/ui/test_jobs_page.py

@@ -55,6 +55,7 @@ class TestJobsPageRendering:
 
         assert response.status_code == 200
         assert "Document Name" in response.text
+        assert "# Sources" in response.text
         assert "Source Filename" not in response.text
         assert "Updated" in response.text
         assert "Created" not in response.text
@@ -112,6 +113,8 @@ class TestJobsPageRendering:
         assert response.status_code == 200
         assert "Create Processing Job" in response.text
         assert "Preselected Journal Entry" in response.text
+        assert "Provider" in response.text
+        assert "Model" in response.text
 
     @pytest.mark.asyncio
     async def test_job_detail_page_renders_logistics_and_links(self, app_client, seed_document_with_unlinked_job):

tests/ui/test_navigation_and_mounts.py

@@ -28,6 +28,7 @@ class TestNavigationAndMounts:
             "/ui/homepage",
             "/ui/homepage/edit",
             "/ui/documents",
+            "/ui/tags",
             "/ui/people",
             "/ui/sources",
             "/ui/jobs",

tests/ui/test_pages_registration.py

@@ -16,6 +16,7 @@ class TestPageRegistration:
         people_response = client.get("/ui/people")
         sources_response = client.get("/ui/sources")
         jobs_response = client.get("/ui/jobs")
+        tags_response = client.get("/ui/tags")
         settings_response = client.get("/ui/settings")
 
         assert homepage_response.status_code == 200
@@ -23,9 +24,11 @@ class TestPageRegistration:
         assert people_response.status_code == 200
         assert sources_response.status_code == 200
         assert jobs_response.status_code == 200
+        assert tags_response.status_code == 200
         assert settings_response.status_code == 200
         assert "Document Types" in settings_response.text
         assert "Person Roles" in settings_response.text
+        assert "Tags" in settings_response.text
         assert "Prompts" in settings_response.text
         assert "Home Page Text" in settings_response.text
         assert "README.md" not in settings_response.text

tests/ui/test_people_page.py

@@ -41,7 +41,28 @@ class TestPeoplePageRendering:
 
         assert response.status_code == 200
         assert "Ada Lovelace" in response.text
-        assert "Ada" in response.text
+        assert "FamilySearch ID" in response.text
+        assert "# Documents" in response.text
+        assert "Display Name" not in response.text
+        assert "Maiden Name" not in response.text
+
+    @pytest.mark.asyncio
+    async def test_people_page_shows_document_counts(self, app_client):
+        _, client = app_client
+
+        async with session_scope() as session:
+            role = (await session.exec(select(PersonRole).where(PersonRole.semantic_key == "author"))).one()
+            person = Person(full_name="Counted Person")
+            document = Document(name="Linked For Count")
+            session.add_all([person, document])
+            await session.flush()
+            session.add(DocumentPerson(document_id=document.id, person_id=person.id, role_id=role.id))
+            await session.commit()
+
+        response = client.get("/ui/people")
+
+        assert response.status_code == 200
+        assert '"document_count":1' in response.text
 
     def test_person_create_page_renders_fields(self, app_client):
         _, client = app_client

tests/ui/test_sources_page.py

@@ -261,9 +261,8 @@ class TestSourcesPageRendering:
         assert "SOURCE PAGE 1: DETAIL-SOURCE.PNG" in response.text.upper()
         assert "SOURCE METADATA" in response.text.upper()
         assert "SOURCEJOB METADATA" in response.text.upper()
-        assert "TRANSCRIPTION TEXT" in response.text.upper()
+        assert "TRANSCRIPTION TEXT" not in response.text.upper()
         assert "EDITABLE REVISION" in response.text.upper()
-        assert "original transcription text" in response.text
         assert "human revision text" in response.text
         assert "Save revision" in response.text
         assert "Previous Page" in response.text

tests/ui/test_tags_page.py

@@ -0,0 +1,43 @@
+"""Tests for the tags page route and grouped filtering behavior."""
+
+import pytest
+
+from transcription.db import session_scope
+from transcription.db.models import Document
+from transcription.services.documents import DocumentService
+
+
+@pytest.mark.integration
+class TestTagsPageRendering:
+    def test_tags_page_renders_empty_state_without_tags(self, app_client):
+        _, client = app_client
+
+        response = client.get("/ui/tags")
+
+        assert response.status_code == 200
+        assert "Tags" in response.text
+        assert "No tags are configured yet." in response.text
+
+    @pytest.mark.asyncio
+    async def test_tags_page_groups_documents_by_tag(self, app_client):
+        app, client = app_client
+        documents = DocumentService(session_factory=app.state.runtime.session_factory)
+
+        async with session_scope(session_factory=app.state.runtime.session_factory) as session:
+            first = Document(name="Tagged Letter")
+            second = Document(name="Tagged Journal")
+            session.add_all([first, second])
+            await session.flush()
+            await documents.sync_document_tags_by_labels(document_id=first.id, labels=["Family"], session=session)
+            await documents.sync_document_tags_by_labels(
+                document_id=second.id,
+                labels=["Family", "Research"],
+                session=session,
+            )
+            await session.commit()
+
+        response = client.get("/ui/tags")
+
+        assert response.status_code == 200
+        assert "Filter by tag" in response.text
+        assert "Tags" in response.text