From 0d554c0648901dbc04db85535c82172e76ab17eb Mon Sep 17 00:00:00 2001 From: Jim Lancaster <40281233+zoltan57@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:32:52 -0500 Subject: [PATCH] V4.11 Added tags + lots of little changes to the UI --- docs/roadmap_plan.md | 51 +++--- docs/schema.md | 36 +++- docs/ui/pages/documents.md | 7 +- docs/ui/pages/jobs.md | 4 +- docs/ui/pages/people.md | 5 +- docs/ui/pages/settings.md | 7 +- docs/ui/pages/sources.md | 7 +- docs/ui/pages/tags.md | 31 ++++ src/transcription/db/migration.py | 2 + src/transcription/db/models.py | 52 ++++++ src/transcription/services/documents.py | 164 ++++++++++++++++++ src/transcription/services/people.py | 3 +- src/transcription/services/workflows.py | 4 + src/transcription/ui/__init__.py | 2 + src/transcription/ui/components/app_shell.py | 3 + .../ui/components/table/documents.py | 14 +- src/transcription/ui/components/table/jobs.py | 17 +- .../ui/components/table/people.py | 35 ++-- src/transcription/ui/pages/documents_page.py | 72 +++++++- src/transcription/ui/pages/jobs_page.py | 21 ++- src/transcription/ui/pages/people_page.py | 4 +- src/transcription/ui/pages/settings_page.py | 117 +++++++++++++ src/transcription/ui/pages/sources_page.py | 46 +++-- src/transcription/ui/pages/tags_page.py | 92 ++++++++++ tests/services/test_document_service.py | 26 +++ tests/services/test_v44_workflows.py | 3 + tests/test_db.py | 15 +- tests/test_storage_reconciliation.py | 116 +++++++++++++ tests/ui/conftest.py | 4 + tests/ui/test_documents_page.py | 12 +- tests/ui/test_jobs_page.py | 3 + tests/ui/test_navigation_and_mounts.py | 1 + tests/ui/test_pages_registration.py | 3 + tests/ui/test_people_page.py | 23 ++- tests/ui/test_sources_page.py | 3 +- tests/ui/test_tags_page.py | 43 +++++ 36 files changed, 932 insertions(+), 116 deletions(-) create mode 100644 docs/ui/pages/tags.md create mode 100644 src/transcription/ui/pages/tags_page.py create mode 100644 tests/test_storage_reconciliation.py create mode 100644 tests/ui/test_tags_page.py diff --git a/docs/roadmap_plan.md b/docs/roadmap_plan.md index 9aa5ee5..078cd6b 100644 --- a/docs/roadmap_plan.md +++ b/docs/roadmap_plan.md @@ -79,29 +79,36 @@ Small, low-risk structural-parity fixes across Person/Job/Document detail pages: --- -## V4.11 — Bigger UI Features +## V4.11 — Approved Scope -- **Tags** (supersedes "collections"): many-to-many tagging for documents, - reusing the Settings registry pattern (autocomplete against existing tags, - managed like Document Types/Person Roles). Since you want dedicated UI to - browse/filter by tag, include a tag-filter view (e.g. a "Tags" entry point - showing documents grouped/filterable by tag) as part of this version, not - deferred — that's the UI surface that makes tags actually replace collections - day-to-day, not just a data field. -- **UI theme selection**: kept simple, per your confirmation — 2–4 curated - themes (e.g. Archival/Sepia, Light, Dark, High-contrast), a single stored - preference, swapped via existing CSS variables/Tailwind tokens in - `theme.py`. No open-ended theme builder. -- **Edit Transcription sub-page**: source image + transcription text + - editable revision side-by-side, keeping the existing Save Revision/Reset - buttons. You noted the exact interaction/layout needs to be clarified when - this is scoped — treat this as a design-first item: confirm the two/three - column breakpoint behavior and what happens on narrow windows before - building. -- **Source detail page reflow** (move Candidate Machine Transcriptions to the - bottom of the image column; align Source Metadata's top edge with - Transcription Text): do this *after* the Edit Transcription sub-page above, - since that page will reuse/rearrange the same boxes — avoids reflowing twice. +- **Tags** (supersedes "collections"): many-to-many tagging for Documents with + Settings-style management (same pattern as Document Types and Person Roles), + autocomplete-capable assignment, and a dedicated **Tags** entry point for + browse/filter-by-tag workflows. +- **Source Detail simplification**: remove the separate **Transcription Text** + card; show Source image + Editable Revision + Source/SourceJob metadata in a + 3-column top layout, then keep Candidate Machine Transcriptions below the + image/revision area. +- **Integrity reconciliation checks in tests**: + - document folder count under `UPLOAD_DIR/documents` must equal `document` + row count. + - source file count under each `UPLOAD_DIR/documents/{document_id}` folder + must equal `source` row count for that Document. + - failures should include actionable mismatch details (missing row/folder or + file/source mapping). +- **UI table updates**: + - Archival Documents: remove **Archive Ref**, add **# Sources**. + - Archival Entities: People: remove **Display Name** and **Maiden Name** + columns, add **FamilySearch ID**. + - Transcription Pipeline Jobs: add **# Sources**. +- **Create Processing Job page**: Provider and Model must be selectable for new + job creation. + +Deferred out of this release: + +- UI theme selection. +- Settings-based `.env` editing and runtime controls. +- Person table structural redesign (removing/splitting name fields). --- diff --git a/docs/schema.md b/docs/schema.md index 738ff3b..1c972e5 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -6,10 +6,11 @@ This document is the field-accurate Version 4 schema contract aligned to `src/tr - `src/transcription/db/models.py:60-78` (status and purpose enums) - `src/transcription/db/models.py:80-120` (`DocumentType`, `PersonRole`) -- `src/transcription/db/models.py:122-207` (`Document`, `Person`, `DocumentPerson`) -- `src/transcription/db/models.py:208-255` (`Job`) -- `src/transcription/db/models.py:273-386` (`Source`, `JobSource`) -- `src/transcription/db/models.py:387-445` (`ExecutionAttempt`) +- `src/transcription/db/models.py:122-148` (`Tag`, `Document`) +- `src/transcription/db/models.py:149-255` (`Person`, `DocumentPerson`, `DocumentTag`) +- `src/transcription/db/models.py:256-303` (`Job`) +- `src/transcription/db/models.py:304-417` (`Source`, `JobSource`) +- `src/transcription/db/models.py:418-476` (`ExecutionAttempt`) ## Entity Relationship Overview @@ -19,8 +20,10 @@ erDiagram Document ||--o{ Job : has Document ||--o{ Source : has Document ||--o{ DocumentPerson : links + Document ||--o{ DocumentTag : tagged Person ||--o{ DocumentPerson : links PersonRole ||--o{ DocumentPerson : labels + Tag ||--o{ DocumentTag : labels Job ||--o{ JobSource : includes Source ||--o{ JobSource : participates JobSource ||--o{ ExecutionAttempt : attempts @@ -74,6 +77,18 @@ erDiagram | `created_at` | `datetime` | default now | | `updated_at` | `datetime` | default now, onupdate | +### `Tag` + +| Field | Type | Notes | +| :--- | :--- | :--- | +| `id` | `UUID` | PK | +| `semantic_key` | `str \| None` | nullable unique, indexed | +| `label` | `str` | required | +| `normalized_label` | `str` | unique, indexed | +| `is_active` | `bool` | default `True` | +| `created_at` | `datetime` | default now | +| `updated_at` | `datetime` | default now, onupdate | + ### `Document` | Field | Type | Notes | @@ -124,6 +139,19 @@ erDiagram Constraint: - `UniqueConstraint(document_id, person_id)` named `uq_document_person` +### `DocumentTag` + +| Field | Type | Notes | +| :--- | :--- | :--- | +| `id` | `UUID` | PK | +| `document_id` | `UUID` | FK -> `document.id`, indexed | +| `tag_id` | `UUID` | FK -> `tag.id`, indexed | +| `created_at` | `datetime` | default now | +| `updated_at` | `datetime` | default now, onupdate | + +Constraint: +- `UniqueConstraint(document_id, tag_id)` named `uq_document_tag` + ### `Job` | Field | Type | Notes | diff --git a/docs/ui/pages/documents.md b/docs/ui/pages/documents.md index acb5538..238bf83 100644 --- a/docs/ui/pages/documents.md +++ b/docs/ui/pages/documents.md @@ -22,9 +22,10 @@ Documents manages the archival record for each historical artifact independently - The title is **Archival Documents**. - **Create new document** opens the create route. - The table defaults to Document Title order and supports search and column sorting. -- Columns are Document Title, Type, Author, Document Date, and Archive Ref. +- Columns are Document Title, Type, Author, Document Date, and # Sources. - Document Title is left-aligned; the remaining columns are centered. - Author lists all linked people in the `author` role. +- # Sources reflects the count of linked Source rows for each Document. - Date display prefers exact date, then approximate date, then `Unknown`. - Selecting a row opens Document Detail. - No records displays `No documents found in repository.` @@ -43,12 +44,14 @@ Optional: - Document location. - Archive identifier. - Notes. +- Tags. - Linked People, with exactly one Person Role per linked Person. Rules: - Exact date must parse as `YYYY-MM-DD`; browser presentation may follow locale. - Existing people appear with disambiguating labels. +- Tag assignment supports selecting existing tags and adding new labels inline. - **Create new person** opens Person creation. - `person_id` may preselect that Person in the author role on Document creation. - An invalid requested Person produces a warning rather than a broken form. @@ -65,7 +68,7 @@ Rules: - The heading shows name, type, and internal ID. - The first Source, when present, appears in the dark-room viewer. -- Archival Metadata shows authors, Document Type, compact Document date, location, and archive identifier. Notes appear in a separate archival-notes block within the same card. +- Archival Metadata shows authors, Document Type, tags, compact Document date, location, and archive identifier. Notes appear in a separate archival-notes block within the same card. - System Logistics shows created and updated timestamps. - Related People are grouped by role and link to Person Detail. - **Sources & Pipeline Jobs** shows counts and actions for filtered Sources, Document Jobs, and adding a Job. diff --git a/docs/ui/pages/jobs.md b/docs/ui/pages/jobs.md index 46696d8..eded8a2 100644 --- a/docs/ui/pages/jobs.md +++ b/docs/ui/pages/jobs.md @@ -19,7 +19,7 @@ Jobs manages transcription processing runs. A Job belongs to one Document, links - The title is **Transcription Pipeline Jobs**. - **Create job** opens Job creation and **Refresh** reloads the table. -- Columns are Job ID, Status, Document Name, Retries, and Updated. +- Columns are Job ID, Status, Document Name, # Sources, Retries, and Updated. - Updated is the primary date/sort field. - Search covers Job ID, document name, and status. - Status is displayed as a semantic status chip. @@ -31,7 +31,7 @@ Jobs manages transcription processing runs. A Job belongs to one Document, links - A Target Document and at least one source file are required. - `document_id` may preselect a Target Document. - If no Documents exist, the page explains the prerequisite and links to Document creation with a return path. -- Provider and Model are optional request overrides. +- Provider and Model are selectable when creating a new Job. - Upload accepts JPEG, PNG, TIFF, and PDF files and supports multiple/folder selection. - The visible upload queue is sorted alphabetically by original filename. - Files can be removed individually or cleared before submission. diff --git a/docs/ui/pages/people.md b/docs/ui/pages/people.md index 3da3c07..22b4c2b 100644 --- a/docs/ui/pages/people.md +++ b/docs/ui/pages/people.md @@ -19,8 +19,9 @@ People manages reusable historical-person records. A Person may appear in many D - The title is **Archival Entities: People**. - **Create new person** opens the create route. - The table defaults to Full Name order and supports search and column sorting. -- Columns are Full Name, Display Name, Maiden Name, Birth Date, and Death Date. -- Full Name is left-aligned; Display Name, Maiden Name, and date columns are centered. +- Columns are Full Name, FamilySearch ID, Birth Date, Death Date, and # Documents. +- Full Name is left-aligned; FamilySearch ID, date columns, and # Documents are centered. +- # Documents reflects how many linked Documents each Person is connected to. - Birth and death values independently prefer exact date, then approximate date, then `Unknown`. - Selecting a row opens Person Detail. - No records displays `No person records found in repository.` diff --git a/docs/ui/pages/settings.md b/docs/ui/pages/settings.md index 03f9ff4..f520603 100644 --- a/docs/ui/pages/settings.md +++ b/docs/ui/pages/settings.md @@ -8,7 +8,7 @@ Settings manages installation-local registries and editable text assets from one | Route | Purpose | | --- | --- | -| `/settings` | Manage Document Types, Person Roles, Prompts, and Home Page Text. | +| `/settings` | Manage Document Types, Person Roles, Tags, Prompts, and Home Page Text. | ## Behavior @@ -16,15 +16,16 @@ Settings manages installation-local registries and editable text assets from one - Configuration surfaces are grouped as tabs: - **Document Types** - **Person Roles** + - **Tags** - **Prompts** - **Home Page Text** -- Document Types and Person Roles support Add/Edit/Delete with existing guardrails. +- Document Types, Person Roles, and Tags support Add/Edit/Delete with existing guardrails. - Prompts exposes only `transcribe_document.md` for editing and restore-from-backup. - Home Page Text edits the same Markdown content rendered on `/homepage`. ## Acceptance Checklist -- `/ui/settings` renders all four tabs. +- `/ui/settings` renders all five tabs. - Registry and prompt workflows keep existing validation and error handling. - Saving Home Page Text persists content for the homepage view. diff --git a/docs/ui/pages/sources.md b/docs/ui/pages/sources.md index f53c71d..1399830 100644 --- a/docs/ui/pages/sources.md +++ b/docs/ui/pages/sources.md @@ -34,13 +34,12 @@ The list accepts optional `document_id` and `job_id` query parameters. Document - **Delete Source** opens the guarded delete route. - Previous and Next navigate only among Sources belonging to the same Document in page order; unavailable boundary actions are disabled. - The media viewer resolves the stored Source path through the configured upload root. -- Transcription Text is read-only and displays the preferred machine projection, with a legacy latest-JobSource - fallback only when no Source projection exists. -- Editable Revision is seeded from an existing revision or the machine transcription. +- The top layout is three columns: Source image, Editable Revision, and Source/SourceJob metadata. +- Editable Revision is seeded from an existing revision or the preferred machine transcription. - Source Metadata shows upload name, stored filename, page number, Document Name, Document ID, and stored path. Source ID appears in the page-header subtitle. - SourceJob Metadata shows latest status, Job ID, execution time, provider, model, prompt, and failure detail. - Revision Logistics shows revised state, last-revised time, and upload time. -- Candidate Machine Transcriptions remains compact until a candidate is expanded, then compares it with the preferred +- Candidate Machine Transcriptions appears below the image/revision area, remains compact until expanded, then compares it with the preferred machine result and requires confirmation before **Use this transcription**. - Candidate promotion does not alter a human revision. Empty states distinguish no machine result from no candidates. - An orientation-normalized artifact appears in evidence only when recognized metadata required a physical rotation. diff --git a/docs/ui/pages/tags.md b/docs/ui/pages/tags.md new file mode 100644 index 0000000..4d9f3b2 --- /dev/null +++ b/docs/ui/pages/tags.md @@ -0,0 +1,31 @@ +# Tags Page Contract + +## Purpose + +Tags provides a dedicated browse/filter entry point for document tagging workflows. + +## Route + +| Route | Purpose | +| --- | --- | +| `/tags` | Browse Documents grouped by Tag and filter to one Tag. | + +## Behavior + +- The page title is **Tags**. +- When no tags exist, the page shows `No tags are configured yet.` +- A Tag filter select allows narrowing to one tag. +- Each rendered group header includes the tag label and document count. +- Document names are clickable and open Document Detail. + +## Acceptance Checklist + +- `/ui/tags` renders successfully from the main navigation. +- Group counts match the number of linked Documents per Tag. +- Filtering hides non-matching tag groups. + +## Implementation Anchors + +- `src/transcription/ui/pages/tags_page.py` +- `src/transcription/services/documents.py` +- `tests/ui/test_tags_page.py` diff --git a/src/transcription/db/migration.py b/src/transcription/db/migration.py index 93600e8..e1364ea 100644 --- a/src/transcription/db/migration.py +++ b/src/transcription/db/migration.py @@ -29,9 +29,11 @@ from transcription.db.engine import get_database_url EXPORT_TABLE_ORDER = ( "document_type", "person_role", + "tag", "document", "person", "document_person", + "document_tag", "job", "source", "job_source", diff --git a/src/transcription/db/models.py b/src/transcription/db/models.py index 182cf91..2789955 100644 --- a/src/transcription/db/models.py +++ b/src/transcription/db/models.py @@ -119,6 +119,28 @@ class PersonRole(SQLModel, table=True): ) +class Tag(SQLModel, table=True): + """Registry of labels that can be attached to Documents.""" + + __tablename__ = "tag" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + semantic_key: str | None = Field(default=None, index=True, unique=True) + label: str + normalized_label: str = Field(index=True, unique=True) + is_active: bool = True + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + updated_at: datetime = Field( + default_factory=lambda: datetime.now(UTC), + sa_column_kwargs={"onupdate": lambda: datetime.now(UTC)}, + ) + + document_tags: list["DocumentTag"] = Relationship( + back_populates="tag_ref", + sa_relationship_kwargs={"lazy": "raise"}, + ) + + class Document(SQLModel, table=True): """An historical document.""" @@ -141,6 +163,10 @@ class Document(SQLModel, table=True): document_people: list["DocumentPerson"] = Relationship( back_populates="document", sa_relationship_kwargs={"lazy": "raise"} ) + document_tags: list["DocumentTag"] = Relationship( + back_populates="document", + sa_relationship_kwargs={"lazy": "raise"}, + ) document_type_ref: Optional["DocumentType"] = Relationship( back_populates="documents", sa_relationship_kwargs={"lazy": "raise"} ) @@ -205,6 +231,32 @@ class DocumentPerson(SQLModel, table=True): ) +class DocumentTag(SQLModel, table=True): + """Associates Documents with Tags.""" + + __tablename__ = "document_tag" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + document_id: UUID = Field(foreign_key="document.id", index=True) + tag_id: UUID = Field(foreign_key="tag.id", index=True) + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + updated_at: datetime = Field( + default_factory=lambda: datetime.now(UTC), + sa_column_kwargs={"onupdate": lambda: datetime.now(UTC)}, + ) + + __table_args__ = (UniqueConstraint("document_id", "tag_id", name="uq_document_tag"),) + + document: Optional["Document"] = Relationship( + back_populates="document_tags", + sa_relationship_kwargs={"lazy": "raise"}, + ) + tag_ref: Optional["Tag"] = Relationship( + back_populates="document_tags", + sa_relationship_kwargs={"lazy": "raise"}, + ) + + class Job(SQLModel, table=True): """A transcription job tied to a single document.""" diff --git a/src/transcription/services/documents.py b/src/transcription/services/documents.py index 71ad5bd..ef5dd29 100644 --- a/src/transcription/services/documents.py +++ b/src/transcription/services/documents.py @@ -20,7 +20,9 @@ from ..db.loading import orm_attribute from ..db.loading import selectinload from ..db.models import Document from ..db.models import DocumentPerson +from ..db.models import DocumentTag from ..db.models import DocumentType +from ..db.models import Tag from ..db.registries import AUTHOR_ROLE_SEMANTIC_KEY from ..errors import AppError from ..errors import ErrorCategory @@ -53,6 +55,10 @@ class DocumentTypeError(DocumentError): """Raised when Document Type maintenance fails.""" +class TagError(DocumentError): + """Raised when Tag maintenance fails.""" + + class DocumentTypeRegistry(RegistryService[DocumentType]): """Document Type registry maintenance.""" @@ -75,6 +81,28 @@ class DocumentTypeRegistry(RegistryService[DocumentType]): type DocumentTypeSummary = RegistrySummary +class TagRegistry(RegistryService[Tag]): + """Tag registry maintenance.""" + + model = Tag + error = TagError + noun = "Tag" + short_noun = "tag" + referenced_retainer = "historical Documents" + + def reference_model(self) -> type[SQLModel]: + return DocumentTag + + def reference_id_column(self) -> Any: + return col(DocumentTag.id) + + def reference_key_column(self) -> Any: + return col(DocumentTag.tag_id) + + +type TagSummary = RegistrySummary + + @dataclass(frozen=True, slots=True) class DocumentPrintSource: id: UUID @@ -119,6 +147,7 @@ class DocumentService(ServiceBase): ) -> None: super().__init__(session_factory, settings) self._document_types = DocumentTypeRegistry(self.session_factory, self.settings) + self._tags = TagRegistry(self.session_factory, self.settings) async def _validate_document_type(self, *, session: AsyncSession, document: Document) -> None: """Validate the UUID-backed Document Type reference.""" @@ -272,6 +301,8 @@ class DocumentService(ServiceBase): selectinload(Document.document_people).selectinload(orm_attribute(DocumentPerson.person)), selectinload(Document.document_people).selectinload(orm_attribute(DocumentPerson.role_ref)), selectinload(Document.document_type_ref), + selectinload(Document.document_tags).selectinload(orm_attribute(DocumentTag.tag_ref)), + selectinload(Document.sources), ) result = await _session.exec(query) return result.all() @@ -287,6 +318,7 @@ class DocumentService(ServiceBase): selectinload(Document.document_people).selectinload(orm_attribute(DocumentPerson.person)), selectinload(Document.document_people).selectinload(orm_attribute(DocumentPerson.role_ref)), selectinload(Document.document_type_ref), + selectinload(Document.document_tags).selectinload(orm_attribute(DocumentTag.tag_ref)), ) .where(Document.id == document_id) .execution_options(populate_existing=True) @@ -362,6 +394,15 @@ class DocumentService(ServiceBase): """List configured document types.""" return await self._document_types.list_entries(active_only=active_only, session=session) + async def list_tags( + self, + *, + active_only: bool = True, + session: AsyncSession | None = None, + ) -> Sequence[Tag]: + """List configured tags.""" + return await self._tags.list_entries(active_only=active_only, session=session) + async def list_document_type_summaries( self, *, @@ -380,6 +421,24 @@ class DocumentService(ServiceBase): for document_type, document_count in rows ] + async def list_tag_summaries( + self, + *, + session: AsyncSession | None = None, + ) -> Sequence[TagSummary]: + """List Tags alphabetically with current usage counts.""" + rows = await self._tags.list_entries_with_counts(session=session) + return [ + RegistrySummary( + id=tag.id, + label=tag.label, + is_active=tag.is_active, + is_built_in=tag.semantic_key is not None, + reference_count=document_count, + ) + for tag, document_count in rows + ] + async def create_document_type( self, *, @@ -390,6 +449,16 @@ class DocumentService(ServiceBase): """Create a UUID-identified Document Type with a unique label.""" return await self._document_types.create_entry(label=label, is_active=is_active, session=session) + async def create_tag( + self, + *, + label: str, + is_active: bool = True, + session: AsyncSession | None = None, + ) -> Tag: + """Create a UUID-identified Tag with a unique label.""" + return await self._tags.create_entry(label=label, is_active=is_active, session=session) + async def read_document_type( self, document_type_id: UUID, @@ -399,6 +468,15 @@ class DocumentService(ServiceBase): """Read a Document Type by id.""" return await self._document_types.read_entry(document_type_id, session=session) + async def read_tag( + self, + tag_id: UUID, + *, + session: AsyncSession | None = None, + ) -> Tag: + """Read a Tag by id.""" + return await self._tags.read_entry(tag_id, session=session) + async def update_document_type( self, document_type_id: UUID, @@ -415,6 +493,22 @@ class DocumentService(ServiceBase): session=session, ) + async def update_tag( + self, + tag_id: UUID, + *, + label: str, + is_active: bool, + session: AsyncSession | None = None, + ) -> Tag: + """Update a Tag label and active state.""" + return await self._tags.update_entry( + tag_id, + label=label, + is_active=is_active, + session=session, + ) + async def delete_document_type( self, document_type_id: UUID, @@ -424,6 +518,15 @@ class DocumentService(ServiceBase): """Delete an unreferenced Document Type without cascade behavior.""" await self._document_types.delete_entry(document_type_id, session=session) + async def delete_tag( + self, + tag_id: UUID, + *, + session: AsyncSession | None = None, + ) -> None: + """Delete an unreferenced Tag without cascade behavior.""" + await self._tags.delete_entry(tag_id, session=session) + async def is_document_type_referenced( self, document_type_id: UUID, @@ -433,6 +536,15 @@ class DocumentService(ServiceBase): """Return whether a Document references a Document Type.""" return await self._document_types.is_referenced(document_type_id, session=session) + async def is_tag_referenced( + self, + tag_id: UUID, + *, + session: AsyncSession | None = None, + ) -> bool: + """Return whether a Document references a Tag.""" + return await self._tags.is_referenced(tag_id, session=session) + async def set_document_type( self, *, @@ -448,6 +560,58 @@ class DocumentService(ServiceBase): await self._finalize(session=_session, caller_session=session, refresh=(document,)) return document + async def sync_document_tags_by_labels( + self, + *, + document_id: UUID, + labels: Sequence[str], + session: AsyncSession | None = None, + ) -> None: + """Replace a Document's tag set using label-based assignment.""" + normalized_labels = [self._tags.normalize_label(label) for label in labels] + deduplicated_labels = list(dict.fromkeys(normalized_labels)) + label_keys = [self._tags.label_key(label) for label in deduplicated_labels] + + async with self._session_scope(session) as _session: + existing_document = await _session.get(Document, document_id) + if existing_document is None: + raise DocumentError( + f"Document with id {document_id} not found", + category=ErrorCategory.NOT_FOUND, + suggestion="Refresh and select an existing document.", + ) + + existing_tags = ( + (await _session.exec(select(Tag).where(col(Tag.normalized_label).in_(label_keys)))) + .all() + if label_keys + else [] + ) + tags_by_key = {tag.normalized_label: tag for tag in existing_tags} + selected_tag_ids: set[UUID] = set() + + for label in deduplicated_labels: + key = self._tags.label_key(label) + tag = tags_by_key.get(key) + if tag is None: + tag = await self._tags.create_entry(label=label, is_active=True, session=_session) + tags_by_key[key] = tag + selected_tag_ids.add(tag.id) + + links = ( + await _session.exec(select(DocumentTag).where(DocumentTag.document_id == document_id)) + ).all() + existing_ids = {link.tag_id for link in links} + + for link in links: + if link.tag_id not in selected_tag_ids: + await _session.delete(link) + + for tag_id in selected_tag_ids - existing_ids: + _session.add(DocumentTag(document_id=document_id, tag_id=tag_id)) + + await self._finalize(session=_session, caller_session=session) + def _print_media_type(filename: str) -> str: """Resolve a stored Source filename to its MIME type for print rendering.""" diff --git a/src/transcription/services/people.py b/src/transcription/services/people.py index 7817eea..ee73865 100644 --- a/src/transcription/services/people.py +++ b/src/transcription/services/people.py @@ -217,7 +217,8 @@ class PeopleService(ServiceBase): async def list_people(self, *, session: AsyncSession | None = None) -> Sequence[Person]: async with self._session_scope(session) as _session: - return (await _session.exec(select(Person))).all() + query = select(Person).options(selectinload(Person.document_people)) + return (await _session.exec(query)).all() async def list_person_roles( self, diff --git a/src/transcription/services/workflows.py b/src/transcription/services/workflows.py index 11aaf60..bc2ace1 100644 --- a/src/transcription/services/workflows.py +++ b/src/transcription/services/workflows.py @@ -44,6 +44,7 @@ async def create_document_with_people( *, document: Document, links: list[DocumentPersonInput], + tag_labels: list[str], documents: DocumentService, people: PeopleService, ) -> Document: @@ -51,6 +52,7 @@ async def create_document_with_people( async with transaction_scope(session_factory=documents.session_factory) as session: created = await documents.create_document(document, session=session) await people.sync_document_people(document_id=created.id, links=links, session=session) + await documents.sync_document_tags_by_labels(document_id=created.id, labels=tag_labels, session=session) return created @@ -58,6 +60,7 @@ async def update_document_with_people( *, document: Document, links: list[DocumentPersonInput], + tag_labels: list[str], documents: DocumentService, people: PeopleService, ) -> Document: @@ -65,6 +68,7 @@ async def update_document_with_people( async with transaction_scope(session_factory=documents.session_factory) as session: updated = await documents.update_document(document, session=session) await people.sync_document_people(document_id=updated.id, links=links, session=session) + await documents.sync_document_tags_by_labels(document_id=updated.id, labels=tag_labels, session=session) return updated diff --git a/src/transcription/ui/__init__.py b/src/transcription/ui/__init__.py index 9e27752..a630209 100644 --- a/src/transcription/ui/__init__.py +++ b/src/transcription/ui/__init__.py @@ -13,6 +13,7 @@ from transcription.ui.pages.people_page import register_page as register_people_ from transcription.ui.pages.print_preview_page import register_page as register_print_preview_page from transcription.ui.pages.settings_page import register_page as register_settings_page from transcription.ui.pages.sources_page import register_page as register_sources_page +from transcription.ui.pages.tags_page import register_page as register_tags_page from transcription.ui.resources import read_css from transcription.ui.theme import apply_archival_theme @@ -37,6 +38,7 @@ def register_pages(app: FastAPI) -> None: _register_global_styles(app) register_home_page() register_documents_page() + register_tags_page() register_people_page() register_print_preview_page() register_sources_page() diff --git a/src/transcription/ui/components/app_shell.py b/src/transcription/ui/components/app_shell.py index dcd6e85..562b180 100644 --- a/src/transcription/ui/components/app_shell.py +++ b/src/transcription/ui/components/app_shell.py @@ -8,6 +8,7 @@ from transcription.ui.resources import read_svg NAV_ITEMS: tuple[tuple[str, str, str], ...] = ( ("Documents", "/documents", "description"), + ("Tags", "/tags", "sell"), ("People", "/people", "group"), ("Sources", "/sources", "folder"), ("Jobs", "/jobs", "work_history"), @@ -22,6 +23,8 @@ def _is_active_path(*, current_path: str, item_path: str) -> bool: return current_path == "/documents" or current_path.startswith("/documents/") if item_path == "/people": return current_path == "/people" or current_path.startswith("/people/") + if item_path == "/tags": + return current_path == "/tags" or current_path.startswith("/tags/") if item_path == "/sources": return current_path == "/sources" or current_path.startswith("/sources/") if item_path == "/settings": diff --git a/src/transcription/ui/components/table/documents.py b/src/transcription/ui/components/table/documents.py index 88da34f..a401814 100644 --- a/src/transcription/ui/components/table/documents.py +++ b/src/transcription/ui/components/table/documents.py @@ -23,7 +23,7 @@ class DocumentTableRow: document_type: str authors: str document_date: str - archive_identifier: str + source_count: int def _serialize_rows(rows: Sequence[DocumentTableRow]) -> list[dict[str, Any]]: @@ -34,7 +34,7 @@ def _serialize_rows(rows: Sequence[DocumentTableRow]) -> list[dict[str, Any]]: "document_type": row.document_type or "Unspecified", "authors": row.authors or "Not set", "document_date": row.document_date, - "archive_identifier": row.archive_identifier or "N/A", + "source_count": row.source_count, } for row in rows ] @@ -87,17 +87,17 @@ def render_documents_table(rows: Sequence[DocumentTableRow]) -> None: "style": "width: 14%;", }, { - "name": "archive_identifier", - "label": "Archive Ref", - "field": "archive_identifier", + "name": "source_count", + "label": "# Sources", + "field": "source_count", "sortable": True, - "classes": "font-mono text-xs", + "classes": "font-mono", "align": "center", "style": "width: 20%;", }, ], default_sort_by="name", - search_placeholder="Search documents by title, type, or reference...", + search_placeholder="Search documents by title, type, or author...", on_row_click_id=lambda doc_id: ui.navigate.to(f"/documents/{doc_id}"), ) diff --git a/src/transcription/ui/components/table/jobs.py b/src/transcription/ui/components/table/jobs.py index eee7130..fa12840 100644 --- a/src/transcription/ui/components/table/jobs.py +++ b/src/transcription/ui/components/table/jobs.py @@ -24,6 +24,7 @@ class JobTableRow: id: UUID status: str document_name: str + source_count: int retry_count: int date_created: str date_updated: str @@ -45,6 +46,7 @@ def _serialize_rows(rows: Sequence[JobTableRow]) -> list[dict[str, Any]]: "id": str(row.id), "status": row.status.lower(), "document_name": row.document_name, + "source_count": row.source_count, "retry_count": row.retry_count, "date_created": _format_timestamp(row.date_created), "date_updated": _format_timestamp(row.date_updated), @@ -71,7 +73,7 @@ def render_jobs_table(rows: Sequence[JobTableRow]) -> None: "field": "id", "sortable": True, "classes": "font-mono text-xs", - "style": "width: 30%;", + "style": "width: 26%;", }, { "name": "status", @@ -79,7 +81,7 @@ def render_jobs_table(rows: Sequence[JobTableRow]) -> None: "field": "status", "sortable": True, "classes": "font-mono", - "style": "width: 15%;", + "style": "width: 13%;", }, { "name": "document_name", @@ -88,7 +90,14 @@ def render_jobs_table(rows: Sequence[JobTableRow]) -> None: "sortable": True, "classes": "font-serif text-left ui-table-cell-wrap", "align": "left", - "style": "width: 35%;", + "style": "width: 31%;", + }, + { + "name": "source_count", + "label": "# Sources", + "field": "source_count", + "sortable": True, + "style": "width: 8%;", }, { "name": "retry_count", @@ -102,7 +111,7 @@ def render_jobs_table(rows: Sequence[JobTableRow]) -> None: "label": "Updated", "field": "updated_sort", "sortable": True, - "style": "width: 12%;", + "style": "width: 14%;", }, ], default_sort_by="updated_sort", diff --git a/src/transcription/ui/components/table/people.py b/src/transcription/ui/components/table/people.py index 6f37797..9f8dfdb 100644 --- a/src/transcription/ui/components/table/people.py +++ b/src/transcription/ui/components/table/people.py @@ -20,10 +20,10 @@ class PersonTableRow: id: UUID full_name: str - display_name: str - maiden_name: str + family_search_id: str birth_date: str death_date: str + document_count: int def _serialize_rows(rows: Sequence[PersonTableRow]) -> list[dict[str, Any]]: @@ -31,10 +31,10 @@ def _serialize_rows(rows: Sequence[PersonTableRow]) -> list[dict[str, Any]]: { "id": str(row.id), "full_name": row.full_name, - "display_name": row.display_name or "Not set", - "maiden_name": row.maiden_name or "N/A", + "family_search_id": row.family_search_id or "Not set", "birth_date": row.birth_date or "Unknown", "death_date": row.death_date or "Unknown", + "document_count": row.document_count, } for row in rows ] @@ -57,22 +57,7 @@ def render_people_table(rows: Sequence[PersonTableRow]) -> None: "sortable": True, "classes": "font-serif font-semibold text-left ui-table-cell-wrap", }, - { - "name": "display_name", - "label": "Display Name", - "field": "display_name", - "sortable": True, - "classes": "ui-table-cell-wrap", - "align": "center", - }, - { - "name": "maiden_name", - "label": "Maiden Name", - "field": "maiden_name", - "sortable": True, - "classes": "ui-table-cell-wrap", - "align": "center", - }, + {"name": "family_search_id", "label": "FamilySearch ID", "field": "family_search_id", "sortable": True}, { "name": "birth_date", "label": "Birth Date", @@ -89,9 +74,17 @@ def render_people_table(rows: Sequence[PersonTableRow]) -> None: "classes": "font-mono text-xs", "align": "center", }, + { + "name": "document_count", + "label": "# Documents", + "field": "document_count", + "sortable": True, + "align": "center", + "classes": "font-mono", + }, ], default_sort_by="full_name", - search_placeholder="Search people by name, birth date, or death date...", + search_placeholder="Search people by name, FamilySearch ID, or dates...", on_row_click_id=lambda person_id: ui.navigate.to(f"/people/{person_id}"), ) diff --git a/src/transcription/ui/pages/documents_page.py b/src/transcription/ui/pages/documents_page.py index d05230b..dd3932e 100644 --- a/src/transcription/ui/pages/documents_page.py +++ b/src/transcription/ui/pages/documents_page.py @@ -10,6 +10,7 @@ from fastapi import Request from fastapi.responses import RedirectResponse from nicegui import ui +from transcription.config import Settings from transcription.db.models import Document from transcription.db.registries import AUTHOR_ROLE_SEMANTIC_KEY from transcription.errors import ErrorCategory @@ -35,12 +36,14 @@ from transcription.ui.components.guards import parsed_record_id from transcription.ui.components.guards import render_record_not_found from transcription.ui.components.linked_people import LinkedPeopleEditor from transcription.ui.components.linked_people import StagedLinkedPerson +from transcription.ui.components.media_urls import resolve_media_url from transcription.ui.components.primitives import destructive_button from transcription.ui.components.primitives import render_empty_state from transcription.ui.components.primitives import section_header_row from transcription.ui.components.table.documents import DocumentTableRow from transcription.ui.components.table.documents import render_documents_table from transcription.ui.components.viewers import dark_room_viewer +from transcription.ui.runtime import resolve_runtime_settings from transcription.ui.theme import page_header from ...db.session import SessionFactoryDep @@ -58,6 +61,7 @@ class DocumentFormFields: location: ui.input archive: ui.input notes: ui.textarea + tags: ui.select def register_page() -> None: # noqa: PLR0915 @@ -75,6 +79,7 @@ def register_page() -> None: # noqa: PLR0915 people = sorted(await people_service.list_people(), key=lambda item: item.full_name.casefold()) role_catalog = list(await people_service.list_person_roles(active_only=False)) type_catalog = await document_service.list_document_types() + tag_catalog = await document_service.list_tags(active_only=True) requested_person_id = parse_uuid(request.query_params.get("person_id")) staged_links: list[StagedLinkedPerson] = [] if requested_person_id is not None and any(person.id == requested_person_id for person in people): @@ -101,6 +106,7 @@ def register_page() -> None: # noqa: PLR0915 ) form = _render_document_form_fields( type_options={str(doc_type.id): doc_type.label for doc_type in type_catalog}, + tag_options=[tag.label for tag in tag_catalog], linked_people=linked_people, ) @@ -137,6 +143,7 @@ def register_page() -> None: # noqa: PLR0915 action=lambda: create_document_with_people( document=candidate, links=linked_people.values(), + tag_labels=_resolve_selected_tag_labels(form.tags.value), documents=document_service, people=people_service, ), @@ -189,16 +196,17 @@ def register_page() -> None: # noqa: PLR0915 document_type=(doc.document_type_ref.label if doc.document_type_ref is not None else ""), authors=", ".join(_author_names(doc)), document_date=compact_date(doc.document_date, doc.document_date_raw), - archive_identifier=doc.archive_identifier or "", + source_count=len(doc.sources), ) for doc in documents ] render_documents_table(rows) @ui.page("/documents/{document_id}") - async def document_detail_page(document_id: str, session_factory: SessionFactoryDep) -> None: + async def document_detail_page(request: Request, document_id: str, session_factory: SessionFactoryDep) -> None: document_service = DocumentService(session_factory=session_factory) render_navigation_header(current_path="/documents") + settings = resolve_runtime_settings(request) parsed_doc_id = parsed_record_id(document_id, noun="Document") if parsed_doc_id is None: @@ -237,7 +245,7 @@ def register_page() -> None: # noqa: PLR0915 ) with ui.grid().classes("w-full grid-cols-12 gap-4"): - _render_bento_viewer_zone(document) + _render_bento_viewer_zone(document, base_url=str(request.base_url), settings=settings) _render_bento_metadata_zone(document) _render_bento_relations_zone(document) @@ -276,6 +284,7 @@ def register_page() -> None: # noqa: PLR0915 people = sorted(await people_service.list_people(), key=lambda item: item.full_name.casefold()) role_catalog = list(await people_service.list_person_roles(active_only=False)) type_catalog = await document_service.list_document_types(active_only=False) + tag_catalog = await document_service.list_tags(active_only=False) linked_people = LinkedPeopleEditor( people=people, roles=role_catalog, @@ -284,6 +293,7 @@ def register_page() -> None: # noqa: PLR0915 form = _render_document_form_fields( document=document, type_options={str(doc_type.id): doc_type.label for doc_type in type_catalog}, + tag_options=[tag.label for tag in tag_catalog], linked_people=linked_people, ) @@ -321,6 +331,7 @@ def register_page() -> None: # noqa: PLR0915 action=lambda: update_document_with_people( document=candidate, links=linked_people.values(), + tag_labels=_resolve_selected_tag_labels(form.tags.value), documents=document_service, people=people_service, ), @@ -410,6 +421,7 @@ def _render_document_form_fields( *, document: Document | None = None, type_options: dict[str, str], + tag_options: list[str], linked_people: LinkedPeopleEditor, ) -> DocumentFormFields: with archival_card(extra_classes="gap-3"): @@ -471,6 +483,26 @@ def _render_document_form_fields( .props("outlined autogrow") .classes("w-full ui-form-surface") ) + selected_tags = ( + sorted( + [ + link.tag_ref.label + for link in (document.document_tags if document is not None else []) + if link.tag_ref is not None + ], + key=str.casefold, + ) + if document is not None + else [] + ) + tags_input = ui.select( + sorted(tag_options, key=str.casefold), + label="Tags", + value=selected_tags, + multiple=True, + with_input=True, + new_value_mode="add-unique", + ).props("outlined use-chips").classes("w-full ui-form-surface") linked_people.render() @@ -483,13 +515,15 @@ def _render_document_form_fields( location=location_input, archive=archive_input, notes=notes_input, + tags=tags_input, ) -def _render_bento_viewer_zone(document: Document) -> None: +def _render_bento_viewer_zone(document: Document, *, base_url: str, settings: Settings) -> None: with ui.column().classes("col-span-12 lg:col-span-4"): source_path = document.sources[0].file_path if document.sources else None - dark_room_viewer(source_path, count_label=f"{len(document.sources)} Source(s) Linked") + source_url = resolve_media_url(source_path, upload_dir=settings.upload_dir, base_url=base_url) + dark_room_viewer(source_url, count_label=f"{len(document.sources)} Source(s) Linked") def _render_bento_metadata_zone(document: Document) -> None: @@ -502,6 +536,11 @@ def _render_bento_metadata_zone(document: Document) -> None: "Document Type:", document.document_type_ref.label if document.document_type_ref is not None else "Not set", ) + tags = sorted( + [link.tag_ref.label for link in document.document_tags if link.tag_ref is not None], + key=str.casefold, + ) + metadata_row("Tags:", ", ".join(tags) if tags else "Not set") metadata_row("Document Date:", compact_date(document.document_date, document.document_date_raw)) metadata_row("Location Created:", document.location_created or "Not set") metadata_row("Archive Identifier:", document.archive_identifier or "Not set") @@ -587,3 +626,26 @@ def _author_names(document: Document) -> list[str]: ), key=str.casefold, ) + + +def _resolve_selected_tag_labels(value: object) -> list[str]: + def flatten(item: object) -> list[str]: + if item is None: + return [] + if isinstance(item, str): + return [item] + if isinstance(item, dict): + if "value" in item: + return flatten(item.get("value")) + if "label" in item: + return flatten(item.get("label")) + return [] + if isinstance(item, (list, tuple, set)): + values: list[str] = [] + for child in item: + values.extend(flatten(child)) + return values + return [str(item)] + + labels = [candidate.strip() for candidate in flatten(value) if candidate and candidate.strip()] + return list(dict.fromkeys(labels)) diff --git a/src/transcription/ui/pages/jobs_page.py b/src/transcription/ui/pages/jobs_page.py index 001ab95..7863c7b 100644 --- a/src/transcription/ui/pages/jobs_page.py +++ b/src/transcription/ui/pages/jobs_page.py @@ -79,6 +79,7 @@ def register_page() -> None: # noqa: PLR0915 id=job.id, status=job.status.value, document_name=job.document.name if job.document is not None else "Unknown document", + source_count=len(job.job_sources), retry_count=job.retry_count, date_created=job.date_created.isoformat(), date_updated=job.date_updated.isoformat(), @@ -146,20 +147,30 @@ def register_page() -> None: # noqa: PLR0915 document_select.value = requested_document_id with ui.row().classes("w-full gap-3 grid grid-cols-1 md:grid-cols-2"): + provider_options = [settings.provider.value] + model_options = list(settings.provider_models) or ([settings.provider_model] if settings.provider_model else []) if locked_source is not None: provider_input = ( - ui.input(label="Provider", value=settings.provider.value) - .props("outlined readonly") + ui.select(provider_options, label="Provider", value=settings.provider.value) + .props("outlined disable") .classes("ui-form-surface") ) model_input = ( - ui.select(list(settings.provider_models), label="Model", value=settings.provider_model) + ui.select(model_options, label="Model", value=settings.provider_model) .props("outlined") .classes("ui-form-surface") ) else: - provider_input = ui.input(label="Provider").props("outlined").classes("ui-form-surface") - model_input = ui.input(label="Model").props("outlined").classes("ui-form-surface") + provider_input = ( + ui.select(provider_options, label="Provider", value=settings.provider.value) + .props("outlined") + .classes("ui-form-surface") + ) + model_input = ( + ui.select(model_options, label="Model", value=settings.provider_model) + .props("outlined use-input") + .classes("ui-form-surface") + ) if locked_source is None: _render_upload_section(uploaded_files) diff --git a/src/transcription/ui/pages/people_page.py b/src/transcription/ui/pages/people_page.py index 5a5d3cd..835b855 100644 --- a/src/transcription/ui/pages/people_page.py +++ b/src/transcription/ui/pages/people_page.py @@ -98,10 +98,10 @@ def register_page() -> None: # noqa: PLR0915 PersonTableRow( id=person.id, full_name=person.full_name, - display_name=person.display_name or "", - maiden_name=person.maiden_name or "", + family_search_id=person.family_search_id or "", birth_date=compact_date(person.birth_date, person.birth_date_raw), death_date=compact_date(person.death_date, person.death_date_raw), + document_count=len(person.document_people), ) for person in people ] diff --git a/src/transcription/ui/pages/settings_page.py b/src/transcription/ui/pages/settings_page.py index c85f2cf..22354fa 100644 --- a/src/transcription/ui/pages/settings_page.py +++ b/src/transcription/ui/pages/settings_page.py @@ -271,6 +271,120 @@ def register_page(*, settings: Settings) -> None: # noqa: PLR0915 ui.button("Edit", icon="edit", on_click=lambda: open_role_editor(creating=False)).props("flat") destructive_button("Delete", icon="delete", on_click=delete_selected_role) + @ui.refreshable + async def render_tags() -> None: # noqa: PLR0915 + with archival_card("Tags"): + ui.label("Tags are listed alphabetically. Select one row to edit or delete it.").classes( + "text-xs ui-text-muted mb-3" + ) + tags_outcome = await run_ui_action( + operation="settings.tags.list", + title="Tags unavailable", + action=documents.list_tag_summaries, + ) + if not tags_outcome.ok: + return + tags = tags_outcome.value or () + if not tags: + render_empty_state("No Tags are configured.", extra_classes="mt-3") + rows = [ + { + "id": str(item.id), + "label": item.label, + "reference_count": item.reference_count, + "is_active": item.is_active, + "is_built_in": item.is_built_in, + } + for item in tags + ] + table = render_registry_table( + rows, + count_field="reference_count", + count_label="Documents", + ) + + async def save_tag( + *, + item_id: UUID | None, + label: str, + is_active: bool, + ) -> bool: + async def _save_tag() -> None: + if item_id is None: + await documents.create_tag(label=label, is_active=is_active) + return + await documents.update_tag( + item_id, + label=label, + is_active=is_active, + ) + + save_outcome = await run_ui_action( + operation="settings.tags.save", + title="Tag save failed", + action=_save_tag, + ) + if not save_outcome.ok: + return False + ui.notify("Tag saved", type="positive") + render_tags.refresh() + return True + + def open_tag_editor(*, creating: bool) -> None: + selected = _selected_table_row(table) + if not creating and selected is None: + ui.notify("Select one Tag to edit.", type="warning") + return + if creating: + item_id = None + current_label = "" + current_active = True + else: + assert selected is not None + item_id = UUID(str(selected["id"])) + current_label = str(selected["label"]) + current_active = bool(selected["is_active"]) + with ui.dialog() as dialog, ui.card().classes("w-full max-w-lg ui-card-surface"): + ui.label("Add Tag" if creating else "Edit Tag").classes("text-lg font-semibold") + label_input = ui.input("Label", value=current_label).props("outlined").classes("w-full") + active_input = ui.checkbox("Active", value=current_active) + + async def submit() -> None: + saved = await save_tag( + item_id=item_id, + label=str(label_input.value or ""), + is_active=bool(active_input.value), + ) + if saved: + dialog.close() + + with ui.row().classes("w-full justify-end gap-2"): + ui.button("Cancel", on_click=dialog.close).props("flat") + ui.button("Save", icon="save", on_click=submit).classes("ui-btn-primary") + dialog.open() + + async def delete_selected_tag() -> None: + selected = _selected_table_row(table) + if selected is None: + ui.notify("Select one Tag to delete.", type="warning") + return + delete_outcome = await run_ui_action( + operation="settings.tags.delete", + title="Tag deletion failed", + action=lambda: documents.delete_tag(UUID(str(selected["id"]))), + ) + if not delete_outcome.ok: + return + ui.notify("Tag deleted", type="positive") + render_tags.refresh() + + with ui.row().classes("w-full items-center gap-2 mt-3"): + ui.button("Add", icon="add", on_click=lambda: open_tag_editor(creating=True)).classes( + "ui-btn-primary" + ) + ui.button("Edit", icon="edit", on_click=lambda: open_tag_editor(creating=False)).props("flat") + destructive_button("Delete", icon="delete", on_click=delete_selected_tag) + @ui.refreshable async def render_prompts() -> None: with archival_card("Prompts"): @@ -381,6 +495,7 @@ def register_page(*, settings: Settings) -> None: # noqa: PLR0915 with ui.tabs().classes("w-full") as tabs: document_types_tab = ui.tab("Document Types") person_roles_tab = ui.tab("Person Roles") + tags_tab = ui.tab("Tags") prompts_tab = ui.tab("Prompts") home_page_text_tab = ui.tab("Home Page Text") @@ -389,6 +504,8 @@ def register_page(*, settings: Settings) -> None: # noqa: PLR0915 await render_document_types() with ui.tab_panel(person_roles_tab): await render_person_roles() + with ui.tab_panel(tags_tab): + await render_tags() with ui.tab_panel(prompts_tab): await render_prompts() with ui.tab_panel(home_page_text_tab): diff --git a/src/transcription/ui/pages/sources_page.py b/src/transcription/ui/pages/sources_page.py index 572ee13..538ea6e 100644 --- a/src/transcription/ui/pages/sources_page.py +++ b/src/transcription/ui/pages/sources_page.py @@ -176,24 +176,26 @@ def register_page() -> None: # noqa: PLR0915 extra_classes="text-xs", ) with ui.grid().classes("w-full grid-cols-12 gap-4"): - with ui.column().classes("col-span-12 lg:col-span-4 gap-2"): - _render_source_navigation(navigation.previous_id, navigation.next_id) - _render_source_viewer_zone( - source, - settings=resolve_runtime_settings(request), - request=request, + with ui.column().classes("col-span-12 lg:col-span-8 gap-4"): + with ui.grid().classes("w-full grid-cols-1 lg:grid-cols-2 gap-4"): + with ui.column().classes("gap-2"): + _render_source_navigation(navigation.previous_id, navigation.next_id) + _render_source_viewer_zone( + source, + settings=resolve_runtime_settings(request), + request=request, + ) + _render_source_transcription_column( + source=source, + original_transcription=original_transcription, + latest_job_source=latest_job_source, + sources_service=sources_service, + ) + _render_machine_candidates( + source=source, + attempts=attempts, + evidence_service=evidence_service, ) - _render_source_transcription_column( - source=source, - original_transcription=original_transcription, - latest_job_source=latest_job_source, - sources_service=sources_service, - ) - _render_machine_candidates( - source=source, - attempts=attempts, - evidence_service=evidence_service, - ) _render_source_metadata_column( source=source, latest_job_source=latest_job_source, @@ -301,7 +303,7 @@ def _render_source_transcription_column( latest_job_source: JobSource | None, sources_service: SourceService, ) -> None: - with ui.column().classes("col-span-12 lg:col-span-4 gap-4"): + with ui.column().classes("gap-4"): _render_source_transcription_zone( source=source, original_transcription=original_transcription, @@ -474,12 +476,6 @@ def _render_source_transcription_zone( latest_job_source: JobSource | None, sources_service: SourceService, ) -> None: - with archival_card(title="Transcription Text"): - if original_transcription: - ui.label(original_transcription).classes("p-2 ui-note-box text-xs whitespace-pre-wrap") - else: - render_empty_state("No transcription text available yet.", italic=True) - with archival_card(title="Editable Revision"): seed_revision = source.revised_text if source.revised_text is not None else (original_transcription or "") revision_input = ( @@ -565,7 +561,7 @@ def _render_machine_candidates( None, ) - with ui.column().classes("col-span-12 lg:col-span-8 gap-2"): # noqa: PLR1702, SIM117 + with ui.column().classes("w-full gap-2"): # noqa: PLR1702, SIM117 with archival_card(title="Candidate Machine Transcriptions"): if preferred_attempt is not None: _render_attempt_warnings(preferred_attempt, label="Preferred machine transcription warnings") diff --git a/src/transcription/ui/pages/tags_page.py b/src/transcription/ui/pages/tags_page.py new file mode 100644 index 0000000..99ef15d --- /dev/null +++ b/src/transcription/ui/pages/tags_page.py @@ -0,0 +1,92 @@ +"""Tags browse and filter page registration.""" + +from __future__ import annotations + +from nicegui import ui + +from transcription.services.documents import DocumentService +from transcription.ui.components.app_shell import render_navigation_header +from transcription.ui.components.cards import archival_card +from transcription.ui.components.error_presenter import run_ui_action +from transcription.ui.components.primitives import render_empty_state +from transcription.ui.components.primitives import section_header_row +from transcription.ui.theme import page_header + +from ...db.session import SessionFactoryDep + + +def register_page() -> None: + """Register the tags browse/filter route.""" + + @ui.page("/tags") + async def tags_page(session_factory: SessionFactoryDep) -> None: + document_service = DocumentService(session_factory=session_factory) + render_navigation_header(current_path="/tags") + + tags_outcome = await run_ui_action( + operation="tags.list", + title="Tags unavailable", + action=document_service.list_tag_summaries, + ) + if not tags_outcome.ok: + return + tag_summaries = tags_outcome.value or () + tag_labels = [item.label for item in tag_summaries] + + documents_outcome = await run_ui_action( + operation="documents.list", + title="Documents unavailable", + action=document_service.list_documents, + ) + if not documents_outcome.ok: + return + documents = documents_outcome.value or () + + with ui.column().classes("w-full max-w-7xl mx-auto p-4 gap-4"): + with section_header_row(): + page_header("Tags", subtitle="Browse documents by tag.") + + if not tag_summaries: + with archival_card(extra_classes="p-8 text-center"): + render_empty_state("No tags are configured yet.") + return + + selected_tag = ( + ui.select(tag_labels, label="Filter by tag") + .props("outlined clearable use-input") + .classes("w-full md:w-96 ui-form-surface") + ) + + @ui.refreshable + def render_groups() -> None: + selected = str(selected_tag.value or "").strip() + with ui.column().classes("w-full gap-3"): + rendered_any = False + for summary in tag_summaries: + if selected and summary.label != selected: + continue + tagged_documents = [ + document + for document in documents + if any( + link.tag_ref is not None and link.tag_ref.id == summary.id + for link in document.document_tags + ) + ] + if not tagged_documents: + continue + rendered_any = True + with archival_card(title=f"{summary.label} ({len(tagged_documents)})"): + for document in sorted(tagged_documents, key=lambda item: item.name.casefold()): + ui.button( + document.name, + on_click=lambda _=None, doc_id=document.id: ui.navigate.to(f"/documents/{doc_id}"), + icon="description", + ).props("flat dense no-caps").classes("self-start ui-link-primary text-xs") + + if not rendered_any: + with archival_card(extra_classes="p-6"): + render_empty_state("No documents match this tag filter.", italic=True) + + selected_tag.on_value_change(lambda _event: render_groups.refresh()) + render_groups() diff --git a/tests/services/test_document_service.py b/tests/services/test_document_service.py index 267e8c1..0bb4d7e 100644 --- a/tests/services/test_document_service.py +++ b/tests/services/test_document_service.py @@ -5,13 +5,16 @@ from datetime import datetime from uuid import uuid4 import pytest +from sqlmodel import select from transcription.config import Settings from transcription.db.models import Document from transcription.db.models import DocumentPerson +from transcription.db.models import DocumentTag from transcription.db.models import Job from transcription.db.models import Person from transcription.db.models import Source +from transcription.db.models import Tag from transcription.services.documents import DocumentDeleteBlockedError from transcription.services.documents import DocumentError from transcription.services.documents import DocumentService @@ -311,3 +314,26 @@ async def test_update_document_person_changes_role_id(default_session_factory): ) assert updated.role_id == recipient_role.id + + +@pytest.mark.asyncio +async def test_sync_document_tags_by_labels_creates_and_replaces_tags(default_session_factory): + documents = DocumentService(session_factory=default_session_factory) + document = await documents.create_document(Document(id=uuid4(), name="tagged-doc")) + + await documents.sync_document_tags_by_labels(document_id=document.id, labels=["Family", "Census"]) + await documents.sync_document_tags_by_labels(document_id=document.id, labels=["Census", "Research"]) + + async with documents._session_scope() as session: + links = (await session.exec(select(DocumentTag).where(DocumentTag.document_id == document.id))).all() + tags = (await session.exec(select(Tag))).all() + + assert len(links) == 2 + linked_ids = {link.tag_id for link in links} + linked_labels = {tag.label for tag in tags if tag.id in linked_ids} + assert linked_labels == {"Census", "Research"} + + listed = await documents.list_documents() + assert len(listed) == 1 + listed_labels = {link.tag_ref.label for link in listed[0].document_tags if link.tag_ref is not None} + assert listed_labels == {"Census", "Research"} diff --git a/tests/services/test_v44_workflows.py b/tests/services/test_v44_workflows.py index af71bc9..9e2c705 100644 --- a/tests/services/test_v44_workflows.py +++ b/tests/services/test_v44_workflows.py @@ -31,6 +31,7 @@ async def test_create_document_with_people_rolls_back_on_invalid_person(default_ await create_document_with_people( document=Document(name="Must roll back"), links=[DocumentPersonInput(person_id=uuid4(), role_id=role.id)], + tag_labels=[], documents=documents, people=people, ) @@ -48,6 +49,7 @@ async def test_update_document_with_people_rolls_back_document_and_links(default document = await create_document_with_people( document=Document(name="Original name"), links=[DocumentPersonInput(person_id=person.id, role_id=role.id)], + tag_labels=[], documents=documents, people=people, ) @@ -63,6 +65,7 @@ async def test_update_document_with_people_rolls_back_document_and_links(default await update_document_with_people( document=candidate, links=[DocumentPersonInput(person_id=person.id, role_id=inactive.id)], + tag_labels=[], documents=documents, people=people, ) diff --git a/tests/test_db.py b/tests/test_db.py index 2890b2d..ac12848 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -41,9 +41,11 @@ async def test_create_all_creates_expected_tables(tmp_path): assert "document" in table_names assert "document_type" in table_names + assert "tag" in table_names assert "person" in table_names assert "person_role" in table_names assert "document_person" in table_names + assert "document_tag" in table_names assert "job" in table_names assert "source" in table_names assert "job_source" in table_names @@ -130,15 +132,19 @@ async def test_create_all_declares_hot_path_indexes(tmp_path): database = inspect(sync_connection) indexes = { table: [index["column_names"] for index in database.get_indexes(table)] - for table in ("job", "source", "job_source", "document", "document_person") + for table in ("job", "source", "job_source", "document", "document_person", "document_tag") } job_source_unique = [ constraint["column_names"] for constraint in database.get_unique_constraints("job_source") ] - return indexes, job_source_unique + document_tag_unique = [ + constraint["column_names"] + for constraint in database.get_unique_constraints("document_tag") + ] + return indexes, job_source_unique, document_tag_unique - indexes, job_source_unique = await connection.run_sync(collect) + indexes, job_source_unique, document_tag_unique = await connection.run_sync(collect) assert ["status", "date_created"] in indexes["job"] assert ["document_id"] in indexes["job"] @@ -150,6 +156,9 @@ async def test_create_all_declares_hot_path_indexes(tmp_path): assert ["document_type_id"] in indexes["document"] for column in ("document_id", "person_id", "role_id"): assert [column] in indexes["document_person"] + assert ["document_id"] in indexes["document_tag"] + assert ["tag_id"] in indexes["document_tag"] + assert ["document_id", "tag_id"] in document_tag_unique finally: await dispose_database_runtime() diff --git a/tests/test_storage_reconciliation.py b/tests/test_storage_reconciliation.py new file mode 100644 index 0000000..af91741 --- /dev/null +++ b/tests/test_storage_reconciliation.py @@ -0,0 +1,116 @@ +"""Integrity checks for document/source filesystem-to-database reconciliation.""" + +from __future__ import annotations + +from pathlib import Path +from uuid import UUID +from uuid import uuid4 + +import pytest +from sqlmodel import func +from sqlmodel import select + +from transcription.config import Settings +from transcription.db import session_scope +from transcription.db.models import Document +from transcription.db.models import Source + + +def _document_folder_ids(root: Path) -> set[str]: + documents_root = root / "documents" + if not documents_root.exists(): + return set() + return {entry.name for entry in documents_root.iterdir() if entry.is_dir()} + + +async def _document_ids(settings: Settings) -> set[str]: + async with session_scope(settings=settings) as session: + rows = await session.exec(select(Document.id)) + return {str(item) for item in rows.all()} + + +async def _source_counts_by_document(settings: Settings) -> dict[str, int]: + async with session_scope(settings=settings) as session: + rows = await session.exec( + select(Source.document_id, func.count(Source.id)).group_by(Source.document_id) + ) + return {str(document_id): int(count) for document_id, count in rows} + + +def _source_file_count_for_document(root: Path, document_id: str) -> int: + directory = root / "documents" / document_id + if not directory.exists(): + return 0 + return sum(1 for entry in directory.iterdir() if entry.is_file()) + + +async def assert_storage_reconciliation(*, upload_dir: Path, settings: Settings) -> None: + folder_ids = _document_folder_ids(upload_dir) + doc_ids = await _document_ids(settings) + + missing_in_table = sorted(folder_ids - doc_ids) + missing_in_folders = sorted(doc_ids - folder_ids) + if missing_in_table: + folder = missing_in_table[0] + raise AssertionError( + "Document directory count and document.doc_id count do not agree. " + f"./data/documents/{folder} does not appear in document table" + ) + if missing_in_folders: + doc_id = missing_in_folders[0] + raise AssertionError( + "Document directory count and document.doc_id count do not agree. " + f"document.doc_id {doc_id} has no corresponding folder in ./data/documents" + ) + + source_counts = await _source_counts_by_document(settings) + for document_id in sorted(doc_ids): + db_count = source_counts.get(document_id, 0) + file_count = _source_file_count_for_document(upload_dir, document_id) + if file_count > db_count: + raise AssertionError( + "Source file count and source.source_id count do not agree. " + f"[UPLOAD_DIR]/documents/{document_id}/ contains file(s) with no source row" + ) + if db_count > file_count: + raise AssertionError( + "Source file count and source.source_id count do not agree. " + f"source.source_id rows exist without files in [UPLOAD_DIR]/documents/{document_id}" + ) + + +@pytest.mark.asyncio +async def test_storage_reconciliation_passes_for_matching_counts(tmp_path, default_settings: Settings): + upload_dir = tmp_path / "uploads" + document_id = str(uuid4()) + document_uuid = UUID(document_id) + file_name = f"{uuid4()}.png" + (upload_dir / "documents" / document_id).mkdir(parents=True, exist_ok=True) + (upload_dir / "documents" / document_id / file_name).write_bytes(b"ok") + + async with session_scope(settings=default_settings) as session: + session.add(Document(id=document_uuid, name="Doc")) + session.add( + Source( + document_id=document_uuid, + page_number=1, + upload_name=file_name, + filename=file_name, + file_path=f"documents/{document_id}/{file_name}", + file_hash="a" * 64, + file_size_bytes=2, + ) + ) + await session.commit() + + await assert_storage_reconciliation(upload_dir=upload_dir, settings=default_settings) + + +@pytest.mark.asyncio +async def test_storage_reconciliation_reports_actionable_mismatch_message(tmp_path, default_settings: Settings): + upload_dir = tmp_path / "uploads" + orphan_dir = upload_dir / "documents" / str(uuid4()) + orphan_dir.mkdir(parents=True, exist_ok=True) + + with pytest.raises(AssertionError, match="document\\.doc_id count do not agree"): + await assert_storage_reconciliation(upload_dir=upload_dir, settings=default_settings) diff --git a/tests/ui/conftest.py b/tests/ui/conftest.py index 502e497..110445c 100644 --- a/tests/ui/conftest.py +++ b/tests/ui/conftest.py @@ -23,6 +23,7 @@ from transcription.db import session as db_session_module from transcription.db import session_scope from transcription.db.models import Document from transcription.db.models import DocumentPerson +from transcription.db.models import DocumentTag from transcription.db.models import ExecutionAttempt from transcription.db.models import Job from transcription.db.models import JobSource @@ -30,6 +31,7 @@ from transcription.db.models import JobSourceStatus from transcription.db.models import JobStatus from transcription.db.models import Person from transcription.db.models import Source +from transcription.db.models import Tag @pytest.fixture(scope="session") @@ -73,11 +75,13 @@ async def clear_ui_database( ) async with session_scope(session_factory=app.state.runtime.session_factory) as session: await session.exec(delete(JobSource)) + await session.exec(delete(DocumentTag)) await session.exec(delete(DocumentPerson)) await session.exec(delete(Source)) await session.exec(delete(Job)) await session.exec(delete(Document)) await session.exec(delete(Person)) + await session.exec(delete(Tag)) await session.commit() diff --git a/tests/ui/test_documents_page.py b/tests/ui/test_documents_page.py index 73f8a88..611507b 100644 --- a/tests/ui/test_documents_page.py +++ b/tests/ui/test_documents_page.py @@ -14,6 +14,7 @@ from transcription.db.models import Job from transcription.db.models import Person from transcription.db.models import PersonRole from transcription.db.models import Source +from transcription.ui.pages.documents_page import _resolve_selected_tag_labels # --- Helper Fixtures --- @@ -81,9 +82,10 @@ class TestDocumentsPageRendering: assert response.status_code == 200 assert "1924 Postcard" in response.text assert "Postcard" in response.text - assert "PC-001" in response.text assert "Document Date" in response.text assert "Author" in response.text + assert "# Sources" in response.text + assert "Archive Ref" not in response.text def test_document_create_page_renders_form(self, app_client): _, client = app_client @@ -95,6 +97,7 @@ class TestDocumentsPageRendering: assert "Document name" in response.text assert "Linked People" in response.text assert "Document type" in response.text + assert "Tags" in response.text @pytest.mark.asyncio async def test_document_create_page_preselects_person_with_disambiguating_label(self, app_client): @@ -209,3 +212,10 @@ class TestDocumentsPageRendering: assert "Delete Document" in response.text assert "Delete document permanently" in response.text assert "Delete is blocked" not in response.text + + +def test_resolve_selected_tag_labels_handles_multiple_payload_shapes(): + assert _resolve_selected_tag_labels("Family") == ["Family"] + assert _resolve_selected_tag_labels(["Family", "Research"]) == ["Family", "Research"] + assert _resolve_selected_tag_labels([{"label": "Family"}, {"value": "Research"}]) == ["Family", "Research"] + assert set(_resolve_selected_tag_labels({"value": {"Family", "Research"}})) == {"Family", "Research"} diff --git a/tests/ui/test_jobs_page.py b/tests/ui/test_jobs_page.py index eb74c78..056eaa5 100644 --- a/tests/ui/test_jobs_page.py +++ b/tests/ui/test_jobs_page.py @@ -55,6 +55,7 @@ class TestJobsPageRendering: assert response.status_code == 200 assert "Document Name" in response.text + assert "# Sources" in response.text assert "Source Filename" not in response.text assert "Updated" in response.text assert "Created" not in response.text @@ -112,6 +113,8 @@ class TestJobsPageRendering: assert response.status_code == 200 assert "Create Processing Job" in response.text assert "Preselected Journal Entry" in response.text + assert "Provider" in response.text + assert "Model" in response.text @pytest.mark.asyncio async def test_job_detail_page_renders_logistics_and_links(self, app_client, seed_document_with_unlinked_job): diff --git a/tests/ui/test_navigation_and_mounts.py b/tests/ui/test_navigation_and_mounts.py index f4434bc..628b156 100644 --- a/tests/ui/test_navigation_and_mounts.py +++ b/tests/ui/test_navigation_and_mounts.py @@ -28,6 +28,7 @@ class TestNavigationAndMounts: "/ui/homepage", "/ui/homepage/edit", "/ui/documents", + "/ui/tags", "/ui/people", "/ui/sources", "/ui/jobs", diff --git a/tests/ui/test_pages_registration.py b/tests/ui/test_pages_registration.py index 09e0ff9..b0655f2 100644 --- a/tests/ui/test_pages_registration.py +++ b/tests/ui/test_pages_registration.py @@ -16,6 +16,7 @@ class TestPageRegistration: people_response = client.get("/ui/people") sources_response = client.get("/ui/sources") jobs_response = client.get("/ui/jobs") + tags_response = client.get("/ui/tags") settings_response = client.get("/ui/settings") assert homepage_response.status_code == 200 @@ -23,9 +24,11 @@ class TestPageRegistration: assert people_response.status_code == 200 assert sources_response.status_code == 200 assert jobs_response.status_code == 200 + assert tags_response.status_code == 200 assert settings_response.status_code == 200 assert "Document Types" in settings_response.text assert "Person Roles" in settings_response.text + assert "Tags" in settings_response.text assert "Prompts" in settings_response.text assert "Home Page Text" in settings_response.text assert "README.md" not in settings_response.text diff --git a/tests/ui/test_people_page.py b/tests/ui/test_people_page.py index 5180647..7ad6507 100644 --- a/tests/ui/test_people_page.py +++ b/tests/ui/test_people_page.py @@ -41,7 +41,28 @@ class TestPeoplePageRendering: assert response.status_code == 200 assert "Ada Lovelace" in response.text - assert "Ada" in response.text + assert "FamilySearch ID" in response.text + assert "# Documents" in response.text + assert "Display Name" not in response.text + assert "Maiden Name" not in response.text + + @pytest.mark.asyncio + async def test_people_page_shows_document_counts(self, app_client): + _, client = app_client + + async with session_scope() as session: + role = (await session.exec(select(PersonRole).where(PersonRole.semantic_key == "author"))).one() + person = Person(full_name="Counted Person") + document = Document(name="Linked For Count") + session.add_all([person, document]) + await session.flush() + session.add(DocumentPerson(document_id=document.id, person_id=person.id, role_id=role.id)) + await session.commit() + + response = client.get("/ui/people") + + assert response.status_code == 200 + assert '"document_count":1' in response.text def test_person_create_page_renders_fields(self, app_client): _, client = app_client diff --git a/tests/ui/test_sources_page.py b/tests/ui/test_sources_page.py index e569ee9..72c3317 100644 --- a/tests/ui/test_sources_page.py +++ b/tests/ui/test_sources_page.py @@ -261,9 +261,8 @@ class TestSourcesPageRendering: assert "SOURCE PAGE 1: DETAIL-SOURCE.PNG" in response.text.upper() assert "SOURCE METADATA" in response.text.upper() assert "SOURCEJOB METADATA" in response.text.upper() - assert "TRANSCRIPTION TEXT" in response.text.upper() + assert "TRANSCRIPTION TEXT" not in response.text.upper() assert "EDITABLE REVISION" in response.text.upper() - assert "original transcription text" in response.text assert "human revision text" in response.text assert "Save revision" in response.text assert "Previous Page" in response.text diff --git a/tests/ui/test_tags_page.py b/tests/ui/test_tags_page.py new file mode 100644 index 0000000..7bae530 --- /dev/null +++ b/tests/ui/test_tags_page.py @@ -0,0 +1,43 @@ +"""Tests for the tags page route and grouped filtering behavior.""" + +import pytest + +from transcription.db import session_scope +from transcription.db.models import Document +from transcription.services.documents import DocumentService + + +@pytest.mark.integration +class TestTagsPageRendering: + def test_tags_page_renders_empty_state_without_tags(self, app_client): + _, client = app_client + + response = client.get("/ui/tags") + + assert response.status_code == 200 + assert "Tags" in response.text + assert "No tags are configured yet." in response.text + + @pytest.mark.asyncio + async def test_tags_page_groups_documents_by_tag(self, app_client): + app, client = app_client + documents = DocumentService(session_factory=app.state.runtime.session_factory) + + async with session_scope(session_factory=app.state.runtime.session_factory) as session: + first = Document(name="Tagged Letter") + second = Document(name="Tagged Journal") + session.add_all([first, second]) + await session.flush() + await documents.sync_document_tags_by_labels(document_id=first.id, labels=["Family"], session=session) + await documents.sync_document_tags_by_labels( + document_id=second.id, + labels=["Family", "Research"], + session=session, + ) + await session.commit() + + response = client.get("/ui/tags") + + assert response.status_code == 200 + assert "Filter by tag" in response.text + assert "Tags" in response.text