From 7eca9fe7dccb2bec736e7cddb97fe248edb75037 Mon Sep 17 00:00:00 2001 From: Jim Lancaster <40281233+zoltan57@users.noreply.github.com> Date: Thu, 3 Sep 2026 05:34:13 -0500 Subject: [PATCH] V6.2 Add GEDCOM data --- .github/instructions/services.instructions.md | 1 + docs/architecture.md | 7 +- docs/requirements.md | 4 +- docs/schema.md | 67 +- docs/ui/pages/settings.md | 1 + pyproject.toml | 1 + src/transcription/db/migration.py | 4 + src/transcription/db/models.py | 153 +++++ src/transcription/services/gedcom_import.py | 599 ++++++++++++++++++ src/transcription/services/maintenance.py | 96 +++ .../ui/components/upload_panel.py | 1 + src/transcription/ui/pages/settings_page.py | 89 ++- tests/fixtures/gedcom/sample_familysearch.ged | 42 ++ tests/services/test_gedcom_import.py | 69 ++ tests/services/test_maintenance_service.py | 25 +- tests/ui/test_pages_registration.py | 1 + uv.lock | 11 + 17 files changed, 1155 insertions(+), 16 deletions(-) create mode 100644 src/transcription/services/gedcom_import.py create mode 100644 tests/fixtures/gedcom/sample_familysearch.ged create mode 100644 tests/services/test_gedcom_import.py diff --git a/.github/instructions/services.instructions.md b/.github/instructions/services.instructions.md index a7f6543..ae7eb29 100644 --- a/.github/instructions/services.instructions.md +++ b/.github/instructions/services.instructions.md @@ -42,6 +42,7 @@ is the only service that may **create or delete** its rows. | `Source`, `JobSource` | `SourceService` | | `Job` | `JobService` | | `Person`, `PersonRole`, `DocumentPerson`, `PersonTag` | `PeopleService` | +| `GenealogyPerson`, `GenealogyFamily`, `GenealogyFamilyChild`, `GenealogyCitation` | `MaintenanceService` | | `Photo` | `PhotosService` | | `MaintenanceRun` | `MaintenanceService` | | `ExecutionAttempt` | `SourceService` | diff --git a/docs/architecture.md b/docs/architecture.md index bc13aa7..9d03359 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -121,6 +121,8 @@ Responsibilities: - `ExecutionAttempt` is append-only evidence for each provider call. - `Photo` is person imagery owned by `PhotosService`. - `MaintenanceRun` is one queued or executed operational maintenance run. +- `GenealogyPerson`, `GenealogyFamily`, `GenealogyFamilyChild`, and `GenealogyCitation` store + imported GEDCOM genealogy data and citation provenance. - `DocumentType` and `PersonRole` are UUID-backed registries with optional protected `semantic_key`. - `Tag` is a shared registry reached through both document and person tagging, linked by `DocumentTag` and `PersonTag`. @@ -147,7 +149,7 @@ Responsibilities: - **MaintenanceRun statuses:** `queued`, `processing`, `succeeded`, `failed` - Maintenance uses `succeeded` rather than `transcribed`; the transcription vocabulary does not apply to operational runs. -- **Maintenance job types:** `backup`, `storage_reconciliation` +- **Maintenance job types:** `backup`, `storage_reconciliation`, `gedcom_import` ## Maintenance Execution @@ -157,7 +159,8 @@ lifetime and is recorded: 1. Settings enqueues a `MaintenanceRun` with `status=queued` and a `triggered_by` marker. 2. The worker claims the oldest queued run with a conditional update, moving it to `processing`. 3. `backup` runs the deploy backup script; `storage_reconciliation` compares stored media against - `Document`/`Source` records. + `Document`/`Source` records; `gedcom_import` parses the latest uploaded `.ged` file and upserts + genealogy records. 4. The run finalizes to `succeeded` or `failed` with summary, timing, log path, and `error_detail`. `MaintenanceRun` records operational history and is not evidence in the `ExecutionAttempt` sense; diff --git a/docs/requirements.md b/docs/requirements.md index da5218d..04dc725 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -53,10 +53,12 @@ are stable. Never renumber an existing ID; retire it explicitly instead. ### Operational Maintenance - **REQ-6-010 Queued Maintenance Runs:** Settings-initiated maintenance must persist a `MaintenanceRun` and execute in the worker, not inline in the request that started it. -- **REQ-6-011 Maintenance Run Types:** `MaintenanceRun.job_type` must use one of `backup`, `storage_reconciliation`. +- **REQ-6-011 Maintenance Run Types:** `MaintenanceRun.job_type` must use one of `backup`, `storage_reconciliation`, `gedcom_import`. - **REQ-6-012 Maintenance Status Lifecycle:** `MaintenanceRun.status` must use one of `queued`, `processing`, `succeeded`, `failed`. - **REQ-6-013 Single Claim:** A queued run must be claimed by at most one worker, using a conditional status update rather than read-then-write. - **REQ-6-014 Run History:** Completed runs must retain status, timing, summary, log reference, and error detail, and expose the log for viewing and download. +- **REQ-6-015 GEDCOM Upload Import:** Settings must support manual `.ged` upload and queue-backed import into genealogy tables. +- **REQ-6-016 GEDCOM Idempotent Upsert:** GEDCOM import must upsert `GenealogyPerson` and `GenealogyFamily` by FamilySearch IDs and avoid duplicate imported citations on re-run. ### Deployment and Runtime Configuration diff --git a/docs/schema.md b/docs/schema.md index 2a3f3c4..8043c71 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -7,7 +7,8 @@ This document is the field-accurate V6.1 schema contract aligned to `src/transcr - `src/transcription/db/models.py` (status and purpose enums, including maintenance lifecycle enums) - `src/transcription/db/models.py:80-120` (`DocumentType`, `PersonRole`) - `src/transcription/db/models.py:122-172` (`Tag`, `Document`) -- `src/transcription/db/models.py:175-281` (`Person`, `Photo`, `DocumentPerson`, `DocumentTag`) +- `src/transcription/db/models.py` (`Person`, `GenealogyPerson`, `GenealogyFamily`, `GenealogyFamilyChild`, `GenealogyCitation`) +- `src/transcription/db/models.py` (`Photo`, `DocumentPerson`, `DocumentTag`) - `src/transcription/db/models.py:285-347` (`Job`) - `src/transcription/db/models.py` (`MaintenanceRun`) - `src/transcription/db/models.py:350-462` (`Source`, `JobSource`) @@ -25,6 +26,13 @@ erDiagram Person ||--o{ DocumentPerson : links Person ||--o{ PersonTag : tagged Person ||--o{ Photo : owns + GenealogyPerson ||--o{ GenealogyFamily : husband + GenealogyPerson ||--o{ GenealogyFamily : wife + GenealogyPerson ||--o{ GenealogyFamilyChild : child + GenealogyFamily ||--o{ GenealogyFamilyChild : includes + GenealogyPerson ||--o{ GenealogyCitation : cited + GenealogyFamily ||--o{ GenealogyCitation : cited + Document ||--o{ GenealogyCitation : evidence PersonRole ||--o{ DocumentPerson : labels Tag ||--o{ DocumentTag : labels Tag ||--o{ PersonTag : labels @@ -62,6 +70,7 @@ erDiagram - `backup` - `storage_reconciliation` +- `gedcom_import` ### MaintenanceRunStatus @@ -142,6 +151,62 @@ erDiagram | `created_at` | `datetime` | default now | | `updated_at` | `datetime` | default now, onupdate | +### `GenealogyPerson` + +| Field | Type | Notes | +| :--- | :--- | :--- | +| `id` | `UUID` | PK | +| `fs_id` | `str` | unique, indexed FamilySearch identifier | +| `full_name` | `str` | required | +| `birth_date` | `date \| None` | optional | +| `birth_date_raw` | `str \| None` | optional | +| `birth_place` | `str \| None` | optional | +| `death_date` | `date \| None` | optional | +| `death_date_raw` | `str \| None` | optional | +| `death_place` | `str \| None` | optional | +| `created_at` | `datetime` | default now | +| `updated_at` | `datetime` | default now, onupdate | + +### `GenealogyFamily` + +| Field | Type | Notes | +| :--- | :--- | :--- | +| `id` | `UUID` | PK | +| `fs_family_id` | `str` | unique, indexed FamilySearch family identifier | +| `husband_id` | `UUID \| None` | nullable FK -> `genealogy_person.id`, indexed | +| `wife_id` | `UUID \| None` | nullable FK -> `genealogy_person.id`, indexed | +| `marriage_date` | `date \| None` | optional | +| `marriage_date_raw` | `str \| None` | optional | +| `marriage_place` | `str \| None` | optional | +| `created_at` | `datetime` | default now | +| `updated_at` | `datetime` | default now, onupdate | + +### `GenealogyFamilyChild` + +| Field | Type | Notes | +| :--- | :--- | :--- | +| `id` | `UUID` | PK | +| `family_id` | `UUID` | FK -> `genealogy_family.id`, indexed | +| `child_id` | `UUID` | FK -> `genealogy_person.id`, indexed | +| `relationship_type` | `str \| None` | optional | +| `created_at` | `datetime` | default now | + +Constraint: +- `UniqueConstraint(family_id, child_id)` named `uq_genealogy_family_child` + +### `GenealogyCitation` + +| Field | Type | Notes | +| :--- | :--- | :--- | +| `id` | `UUID` | PK | +| `genealogy_person_id` | `UUID \| None` | nullable FK -> `genealogy_person.id`, indexed | +| `genealogy_family_id` | `UUID \| None` | nullable FK -> `genealogy_family.id`, indexed | +| `fact_type` | `GenealogyCitationFactType` | enum: `birth`, `death`, `marriage`, `other` | +| `raw_citation_text` | `str` | required raw GEDCOM citation text | +| `source_kind` | `GenealogyCitationSourceKind` | enum: `familysearch_imported`, `transcription_evidence` | +| `document_id` | `UUID \| None` | nullable FK -> `document.id`, indexed | +| `created_at` | `datetime` | default now | + ### `Photo` | Field | Type | Notes | diff --git a/docs/ui/pages/settings.md b/docs/ui/pages/settings.md index 8db27f5..92bf7e2 100644 --- a/docs/ui/pages/settings.md +++ b/docs/ui/pages/settings.md @@ -36,6 +36,7 @@ Settings manages installation-local registries, safe runtime .env settings, and - Prompts exposes only `transcribe_document.md` for editing and restore-from-backup. - Home Page Text edits the same Markdown content rendered on `/homepage`. - Maintenance provides queue-backed **Run Backup** and **Run Storage Reconciliation** actions. +- Maintenance also provides GEDCOM upload and **Run GEDCOM Import** actions, using the same queue-backed `MaintenanceRun` history/log flow. - Maintenance run history shows job type, status, started/finished timestamps, duration, summary, and log view/download actions. - Maintenance actions enqueue work and signal the worker; the page itself does not execute shell commands directly. diff --git a/pyproject.toml b/pyproject.toml index 766fef6..60cbf9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "psycopg2-binary>=2.9.12", "pydantic>=2.13.4", "pydantic-settings>=2.9.1", + "python-gedcom>=1.1.0", "sqlmodel>=0.0.25", ] diff --git a/src/transcription/db/migration.py b/src/transcription/db/migration.py index 3d57dac..0e624a9 100644 --- a/src/transcription/db/migration.py +++ b/src/transcription/db/migration.py @@ -39,6 +39,8 @@ EXPORT_TABLE_ORDER = ( "tag", "document", "person", + "genealogy_person", + "genealogy_family", "photo", "document_person", "document_tag", @@ -47,6 +49,8 @@ EXPORT_TABLE_ORDER = ( "source", "job_source", "execution_attempt", + "genealogy_family_child", + "genealogy_citation", ) BYTES_FIELDS = {"transport_body"} diff --git a/src/transcription/db/models.py b/src/transcription/db/models.py index d2dbb0d..202c912 100644 --- a/src/transcription/db/models.py +++ b/src/transcription/db/models.py @@ -85,6 +85,7 @@ class JobPurpose(StrEnum): class MaintenanceJobType(StrEnum): BACKUP = "backup" STORAGE_RECONCILIATION = "storage_reconciliation" + GEDCOM_IMPORT = "gedcom_import" class MaintenanceRunStatus(StrEnum): @@ -94,6 +95,18 @@ class MaintenanceRunStatus(StrEnum): FAILED = "failed" +class GenealogyCitationFactType(StrEnum): + BIRTH = "birth" + DEATH = "death" + MARRIAGE = "marriage" + OTHER = "other" + + +class GenealogyCitationSourceKind(StrEnum): + FAMILYSEARCH_IMPORTED = "familysearch_imported" + TRANSCRIPTION_EVIDENCE = "transcription_evidence" + + class DocumentType(SQLModel, table=True): """Registry of allowed document types.""" @@ -235,6 +248,146 @@ class Person(SQLModel, table=True): return f"{self.given_names} {self.last_name}".strip() +class GenealogyPerson(SQLModel, table=True): + """An individual imported from a GEDCOM export.""" + + __tablename__ = "genealogy_person" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + fs_id: str = Field(index=True, unique=True) + full_name: str + birth_date: date | None = None + birth_date_raw: str | None = None + birth_place: str | None = None + death_date: date | None = None + death_date_raw: str | None = None + death_place: str | None = None + created_at: datetime = Field(default_factory=_utc_now_naive) + updated_at: datetime = Field( + default_factory=_utc_now_naive, + sa_column_kwargs={"onupdate": _utc_now_naive}, + ) + + husband_families: list["GenealogyFamily"] = Relationship( + back_populates="husband", + sa_relationship_kwargs={"lazy": "raise", "foreign_keys": "[GenealogyFamily.husband_id]"}, + ) + wife_families: list["GenealogyFamily"] = Relationship( + back_populates="wife", + sa_relationship_kwargs={"lazy": "raise", "foreign_keys": "[GenealogyFamily.wife_id]"}, + ) + child_family_memberships: list["GenealogyFamilyChild"] = Relationship( + back_populates="child", + sa_relationship_kwargs={"lazy": "raise"}, + ) + citations: list["GenealogyCitation"] = Relationship( + back_populates="genealogy_person", + sa_relationship_kwargs={"lazy": "raise"}, + ) + + +class GenealogyFamily(SQLModel, table=True): + """A family linking two GenealogyPerson records.""" + + __tablename__ = "genealogy_family" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + fs_family_id: str = Field(index=True, unique=True) + husband_id: UUID | None = Field(default=None, foreign_key="genealogy_person.id", index=True) + wife_id: UUID | None = Field(default=None, foreign_key="genealogy_person.id", index=True) + marriage_date: date | None = None + marriage_date_raw: str | None = None + marriage_place: str | None = None + created_at: datetime = Field(default_factory=_utc_now_naive) + updated_at: datetime = Field( + default_factory=_utc_now_naive, + sa_column_kwargs={"onupdate": _utc_now_naive}, + ) + + husband: Optional["GenealogyPerson"] = Relationship( + back_populates="husband_families", + sa_relationship_kwargs={"lazy": "raise", "foreign_keys": "[GenealogyFamily.husband_id]"}, + ) + wife: Optional["GenealogyPerson"] = Relationship( + back_populates="wife_families", + sa_relationship_kwargs={"lazy": "raise", "foreign_keys": "[GenealogyFamily.wife_id]"}, + ) + children: list["GenealogyFamilyChild"] = Relationship( + back_populates="family", + sa_relationship_kwargs={"lazy": "raise"}, + ) + citations: list["GenealogyCitation"] = Relationship( + back_populates="genealogy_family", + sa_relationship_kwargs={"lazy": "raise"}, + ) + + +class GenealogyFamilyChild(SQLModel, table=True): + """Junction table for child membership within a genealogy family.""" + + __tablename__ = "genealogy_family_child" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + family_id: UUID = Field(foreign_key="genealogy_family.id", index=True) + child_id: UUID = Field(foreign_key="genealogy_person.id", index=True) + relationship_type: str | None = None + created_at: datetime = Field(default_factory=_utc_now_naive) + + __table_args__ = (UniqueConstraint("family_id", "child_id", name="uq_genealogy_family_child"),) + + family: Optional["GenealogyFamily"] = Relationship( + back_populates="children", + sa_relationship_kwargs={"lazy": "raise"}, + ) + child: Optional["GenealogyPerson"] = Relationship( + back_populates="child_family_memberships", + sa_relationship_kwargs={"lazy": "raise"}, + ) + + +class GenealogyCitation(SQLModel, table=True): + """A source citation attached to a genealogical fact.""" + + __tablename__ = "genealogy_citation" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + genealogy_person_id: UUID | None = Field(default=None, foreign_key="genealogy_person.id", index=True) + genealogy_family_id: UUID | None = Field(default=None, foreign_key="genealogy_family.id", index=True) + fact_type: GenealogyCitationFactType = Field( + sa_column=Column( + SAEnum( + GenealogyCitationFactType, + values_callable=lambda enum_cls: [item.value for item in enum_cls], + native_enum=False, + ), + nullable=False, + ) + ) + raw_citation_text: str + source_kind: GenealogyCitationSourceKind = Field( + sa_column=Column( + SAEnum( + GenealogyCitationSourceKind, + values_callable=lambda enum_cls: [item.value for item in enum_cls], + native_enum=False, + ), + nullable=False, + ) + ) + document_id: UUID | None = Field(default=None, foreign_key="document.id", index=True) + created_at: datetime = Field(default_factory=_utc_now_naive) + + genealogy_person: Optional["GenealogyPerson"] = Relationship( + back_populates="citations", + sa_relationship_kwargs={"lazy": "raise"}, + ) + genealogy_family: Optional["GenealogyFamily"] = Relationship( + back_populates="citations", + sa_relationship_kwargs={"lazy": "raise"}, + ) + document: Optional["Document"] = Relationship(sa_relationship_kwargs={"lazy": "raise"}) + + class Photo(SQLModel, table=True): """A reusable image record for Person and homepage galleries.""" diff --git a/src/transcription/services/gedcom_import.py b/src/transcription/services/gedcom_import.py new file mode 100644 index 0000000..a562a7e --- /dev/null +++ b/src/transcription/services/gedcom_import.py @@ -0,0 +1,599 @@ +"""GEDCOM parsing and import helpers for maintenance-driven genealogy sync.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date +from pathlib import Path +from uuid import UUID +from uuid import uuid4 + +from gedcom.element.element import Element +from gedcom.parser import Parser +from sqlalchemy import or_ +from sqlmodel import col +from sqlmodel import delete +from sqlmodel import select +from sqlmodel.ext.asyncio.session import AsyncSession + +from transcription.db.models import GenealogyCitation +from transcription.db.models import GenealogyCitationFactType +from transcription.db.models import GenealogyCitationSourceKind +from transcription.db.models import GenealogyFamily +from transcription.db.models import GenealogyFamilyChild +from transcription.db.models import GenealogyPerson +from transcription.errors import AppError +from transcription.errors import ErrorCategory + +_MONTHS = { + "JAN": 1, + "FEB": 2, + "MAR": 3, + "APR": 4, + "MAY": 5, + "JUN": 6, + "JUL": 7, + "AUG": 8, + "SEP": 9, + "OCT": 10, + "NOV": 11, + "DEC": 12, +} + + +@dataclass(frozen=True, slots=True) +class ParsedCitation: + fact_type: GenealogyCitationFactType + raw_citation_text: str + + +@dataclass(frozen=True, slots=True) +class ParsedPerson: + pointer: str + fs_id: str | None + full_name: str + birth_date: date | None + birth_date_raw: str | None + birth_place: str | None + death_date: date | None + death_date_raw: str | None + death_place: str | None + citations: tuple[ParsedCitation, ...] + + +@dataclass(frozen=True, slots=True) +class ParsedFamilyChild: + child_pointer: str + relationship_type: str | None + + +@dataclass(frozen=True, slots=True) +class ParsedFamily: + fs_family_id: str | None + husband_pointer: str | None + wife_pointer: str | None + marriage_date: date | None + marriage_date_raw: str | None + marriage_place: str | None + children: tuple[ParsedFamilyChild, ...] + citations: tuple[ParsedCitation, ...] + + +@dataclass(frozen=True, slots=True) +class ParsedGedcom: + people: tuple[ParsedPerson, ...] + families: tuple[ParsedFamily, ...] + + +@dataclass(frozen=True, slots=True) +class GedcomImportResult: + new_people: int + updated_people: int + skipped_people_without_fs_id: int + new_families: int + updated_families: int + skipped_families_without_fs_id: int + family_children: int + citations: int + + +class GedcomImportError(AppError): + """Raised when GEDCOM content cannot be parsed or imported.""" + + +def parse_gedcom(*, file_path: Path) -> ParsedGedcom: + parser = Parser() + try: + parser.parse_file(str(file_path)) + except Exception as exc: + raise GedcomImportError( + "GEDCOM file could not be parsed.", + category=ErrorCategory.VALIDATION, + suggestion="Upload a GEDCOM 5.5.1-compatible export and retry.", + detail=f"{type(exc).__name__}: {exc}", + ) from exc + + people: list[ParsedPerson] = [] + families: list[ParsedFamily] = [] + for element in parser.get_root_child_elements(): + tag = element.get_tag() + if tag == "INDI": + people.append(_parse_person(element)) + elif tag == "FAM": + families.append(_parse_family(element)) + return ParsedGedcom(people=tuple(people), families=tuple(families)) + + +async def import_gedcom_file(*, session: AsyncSession, file_path: Path) -> GedcomImportResult: + parsed = parse_gedcom(file_path=file_path) + pointer_to_fs_id = _pointer_to_fs_id_map(parsed=parsed) + people_by_fs_id, new_people, updated_people, skipped_people_without_fs_id = await _upsert_people( + session=session, + parsed=parsed, + ) + ( + families_by_fs_id, + new_families, + updated_families, + skipped_families_without_fs_id, + family_children, + ) = await _upsert_families( + session=session, + parsed=parsed, + pointer_to_fs_id=pointer_to_fs_id, + people_by_fs_id=people_by_fs_id, + ) + + citation_rows = _citation_rows( + parsed=parsed, + pointer_to_fs_id=pointer_to_fs_id, + people=people_by_fs_id, + families=families_by_fs_id, + ) + await _replace_imported_citations( + session=session, + person_ids={item.id for item in people_by_fs_id.values()}, + family_ids={item.id for item in families_by_fs_id.values()}, + citations=citation_rows, + ) + await session.commit() + return GedcomImportResult( + new_people=new_people, + updated_people=updated_people, + skipped_people_without_fs_id=skipped_people_without_fs_id, + new_families=new_families, + updated_families=updated_families, + skipped_families_without_fs_id=skipped_families_without_fs_id, + family_children=family_children, + citations=len(citation_rows), + ) + + +async def _load_people_by_fs_id(*, session: AsyncSession, fs_ids: set[str]) -> dict[str, GenealogyPerson]: + if not fs_ids: + return {} + query = select(GenealogyPerson).where(col(GenealogyPerson.fs_id).in_(fs_ids)) + return {person.fs_id: person for person in (await session.exec(query)).all()} + + +async def _load_families_by_fs_id(*, session: AsyncSession, fs_family_ids: set[str]) -> dict[str, GenealogyFamily]: + if not fs_family_ids: + return {} + query = select(GenealogyFamily).where(col(GenealogyFamily.fs_family_id).in_(fs_family_ids)) + return {family.fs_family_id: family for family in (await session.exec(query)).all()} + + +def _pointer_to_fs_id_map(*, parsed: ParsedGedcom) -> dict[str, str]: + return {person.pointer: person.fs_id for person in parsed.people if person.pointer and person.fs_id is not None} + + +async def _upsert_people( + *, + session: AsyncSession, + parsed: ParsedGedcom, +) -> tuple[dict[str, GenealogyPerson], int, int, int]: + people_with_fs_id = [person for person in parsed.people if person.fs_id is not None] + fs_ids = {person.fs_id for person in people_with_fs_id if person.fs_id is not None} + people_by_fs_id = await _load_people_by_fs_id(session=session, fs_ids=fs_ids) + new_people = 0 + updated_people = 0 + + for person in people_with_fs_id: + assert person.fs_id is not None + existing = people_by_fs_id.get(person.fs_id) + if existing is None: + existing = GenealogyPerson( + fs_id=person.fs_id, + full_name=person.full_name, + birth_date=person.birth_date, + birth_date_raw=person.birth_date_raw, + birth_place=person.birth_place, + death_date=person.death_date, + death_date_raw=person.death_date_raw, + death_place=person.death_place, + ) + session.add(existing) + await session.flush() + people_by_fs_id[person.fs_id] = existing + new_people += 1 + continue + if _apply_person_updates(existing=existing, person=person): + updated_people += 1 + + skipped = len(parsed.people) - len(people_with_fs_id) + return people_by_fs_id, new_people, updated_people, skipped + + +def _apply_person_updates(*, existing: GenealogyPerson, person: ParsedPerson) -> bool: + changed = False + fields = ( + ("full_name", person.full_name), + ("birth_date", person.birth_date), + ("birth_date_raw", person.birth_date_raw), + ("birth_place", person.birth_place), + ("death_date", person.death_date), + ("death_date_raw", person.death_date_raw), + ("death_place", person.death_place), + ) + for name, value in fields: + if getattr(existing, name) != value: + setattr(existing, name, value) + changed = True + return changed + + +async def _upsert_families( + *, + session: AsyncSession, + parsed: ParsedGedcom, + pointer_to_fs_id: Mapping[str, str], + people_by_fs_id: dict[str, GenealogyPerson], +) -> tuple[dict[str, GenealogyFamily], int, int, int, int]: + families_with_fs_id = [family for family in parsed.families if family.fs_family_id is not None] + fs_family_ids = {family.fs_family_id for family in families_with_fs_id if family.fs_family_id is not None} + families_by_fs_id = await _load_families_by_fs_id(session=session, fs_family_ids=fs_family_ids) + new_families = 0 + updated_families = 0 + family_children = 0 + + for family in families_with_fs_id: + assert family.fs_family_id is not None + husband_id = _person_id_from_pointer( + pointer=family.husband_pointer, + pointer_to_fs_id=pointer_to_fs_id, + people=people_by_fs_id, + ) + wife_id = _person_id_from_pointer( + pointer=family.wife_pointer, + pointer_to_fs_id=pointer_to_fs_id, + people=people_by_fs_id, + ) + target = families_by_fs_id.get(family.fs_family_id) + if target is None: + target = GenealogyFamily( + fs_family_id=family.fs_family_id, + husband_id=husband_id, + wife_id=wife_id, + marriage_date=family.marriage_date, + marriage_date_raw=family.marriage_date_raw, + marriage_place=family.marriage_place, + ) + session.add(target) + await session.flush() + families_by_fs_id[family.fs_family_id] = target + new_families += 1 + elif _apply_family_updates(existing=target, family=family, husband_id=husband_id, wife_id=wife_id): + updated_families += 1 + + family_children += await _replace_family_children( + session=session, + family=family, + family_id=target.id, + pointer_to_fs_id=pointer_to_fs_id, + people=people_by_fs_id, + ) + + skipped = len(parsed.families) - len(families_with_fs_id) + return families_by_fs_id, new_families, updated_families, skipped, family_children + + +def _apply_family_updates( + *, + existing: GenealogyFamily, + family: ParsedFamily, + husband_id: UUID | None, + wife_id: UUID | None, +) -> bool: + changed = False + fields = ( + ("husband_id", husband_id), + ("wife_id", wife_id), + ("marriage_date", family.marriage_date), + ("marriage_date_raw", family.marriage_date_raw), + ("marriage_place", family.marriage_place), + ) + for name, value in fields: + if getattr(existing, name) != value: + setattr(existing, name, value) + changed = True + return changed + + +async def _replace_family_children( + *, + session: AsyncSession, + family: ParsedFamily, + family_id: UUID, + pointer_to_fs_id: Mapping[str, str], + people: dict[str, GenealogyPerson], +) -> int: + await session.exec(delete(GenealogyFamilyChild).where(col(GenealogyFamilyChild.family_id) == family_id)) + child_rows = _family_child_rows( + family=family, + family_id=family_id, + pointer_to_fs_id=pointer_to_fs_id, + people=people, + ) + for child_row in child_rows: + session.add(child_row) + return len(child_rows) + + +def _person_id_from_pointer( + *, + pointer: str | None, + pointer_to_fs_id: Mapping[str, str], + people: dict[str, GenealogyPerson], +) -> UUID | None: + if pointer is None: + return None + fs_id = pointer_to_fs_id.get(pointer) + if fs_id is None: + return None + person = people.get(fs_id) + return person.id if person is not None else None + + +def _family_child_rows( + *, + family: ParsedFamily, + family_id: UUID, + pointer_to_fs_id: Mapping[str, str], + people: dict[str, GenealogyPerson], +) -> list[GenealogyFamilyChild]: + rows: list[GenealogyFamilyChild] = [] + seen_child_ids: set[UUID] = set() + for child in family.children: + child_id = _person_id_from_pointer( + pointer=child.child_pointer, + pointer_to_fs_id=pointer_to_fs_id, + people=people, + ) + if child_id is None or child_id in seen_child_ids: + continue + seen_child_ids.add(child_id) + rows.append( + GenealogyFamilyChild( + id=uuid4(), + family_id=family_id, + child_id=child_id, + relationship_type=child.relationship_type, + ) + ) + return rows + + +def _citation_rows( + *, + parsed: ParsedGedcom, + pointer_to_fs_id: Mapping[str, str], + people: dict[str, GenealogyPerson], + families: dict[str, GenealogyFamily], +) -> list[GenealogyCitation]: + rows: list[GenealogyCitation] = [] + seen: set[tuple[UUID | None, UUID | None, str, str]] = set() + + for person in parsed.people: + fs_id = pointer_to_fs_id.get(person.pointer) + if fs_id is None: + continue + person_row = people.get(fs_id) + if person_row is None: + continue + for citation in person.citations: + key = (person_row.id, None, citation.fact_type.value, citation.raw_citation_text) + if key in seen: + continue + seen.add(key) + rows.append( + GenealogyCitation( + id=uuid4(), + genealogy_person_id=person_row.id, + genealogy_family_id=None, + fact_type=citation.fact_type, + raw_citation_text=citation.raw_citation_text, + source_kind=GenealogyCitationSourceKind.FAMILYSEARCH_IMPORTED, + document_id=None, + ) + ) + + for family in parsed.families: + if family.fs_family_id is None: + continue + family_row = families.get(family.fs_family_id) + if family_row is None: + continue + for citation in family.citations: + key = (None, family_row.id, citation.fact_type.value, citation.raw_citation_text) + if key in seen: + continue + seen.add(key) + rows.append( + GenealogyCitation( + id=uuid4(), + genealogy_person_id=None, + genealogy_family_id=family_row.id, + fact_type=citation.fact_type, + raw_citation_text=citation.raw_citation_text, + source_kind=GenealogyCitationSourceKind.FAMILYSEARCH_IMPORTED, + document_id=None, + ) + ) + return rows + + +async def _replace_imported_citations( + *, + session: AsyncSession, + person_ids: set[UUID], + family_ids: set[UUID], + citations: list[GenealogyCitation], +) -> None: + where_clauses = [] + if person_ids: + where_clauses.append(col(GenealogyCitation.genealogy_person_id).in_(person_ids)) + if family_ids: + where_clauses.append(col(GenealogyCitation.genealogy_family_id).in_(family_ids)) + if where_clauses: + target_scope = where_clauses[0] if len(where_clauses) == 1 else or_(*where_clauses) + await session.exec( + delete(GenealogyCitation).where( + col(GenealogyCitation.source_kind) == GenealogyCitationSourceKind.FAMILYSEARCH_IMPORTED, + target_scope, + ) + ) + for citation in citations: + session.add(citation) + + +def _parse_person(element: Element) -> ParsedPerson: + birth_event = _first_child(element, "BIRT") + death_event = _first_child(element, "DEAT") + birth_date_raw = _child_value(birth_event, "DATE") + death_date_raw = _child_value(death_event, "DATE") + return ParsedPerson( + pointer=element.get_pointer() or "", + fs_id=_extract_fs_identifier(element), + full_name=_person_name(element), + birth_date=_parse_exact_date(birth_date_raw), + birth_date_raw=birth_date_raw, + birth_place=_child_value(birth_event, "PLAC"), + death_date=_parse_exact_date(death_date_raw), + death_date_raw=death_date_raw, + death_place=_child_value(death_event, "PLAC"), + citations=( + *_fact_citations(fact_element=birth_event, fact_type=GenealogyCitationFactType.BIRTH), + *_fact_citations(fact_element=death_event, fact_type=GenealogyCitationFactType.DEATH), + ), + ) + + +def _parse_family(element: Element) -> ParsedFamily: + marriage_event = _first_child(element, "MARR") + marriage_date_raw = _child_value(marriage_event, "DATE") + children = tuple( + ParsedFamilyChild( + child_pointer=(child.get_value() or "").strip(), + relationship_type=_child_value(child, "PEDI"), + ) + for child in _children(element, "CHIL") + if (child.get_value() or "").strip() + ) + return ParsedFamily( + fs_family_id=_extract_fs_identifier(element), + husband_pointer=(_child_value(element, "HUSB") or "").strip() or None, + wife_pointer=(_child_value(element, "WIFE") or "").strip() or None, + marriage_date=_parse_exact_date(marriage_date_raw), + marriage_date_raw=marriage_date_raw, + marriage_place=_child_value(marriage_event, "PLAC"), + children=children, + citations=_fact_citations(fact_element=marriage_event, fact_type=GenealogyCitationFactType.MARRIAGE), + ) + + +def _fact_citations( + *, fact_element: Element | None, fact_type: GenealogyCitationFactType +) -> tuple[ParsedCitation, ...]: + if fact_element is None: + return () + citations: list[ParsedCitation] = [] + for source in _children(fact_element, "SOUR"): + raw_citation = _flatten_tag_values(source) + if raw_citation is None: + continue + citations.append(ParsedCitation(fact_type=fact_type, raw_citation_text=raw_citation)) + return tuple(citations) + + +def _extract_fs_identifier(element: Element) -> str | None: + for tag in ("_FSFTID", "FSFTID"): + value = _child_value(element, tag) + if value: + return value + for refn in _children(element, "REFN"): + refn_value = (refn.get_value() or "").strip() + refn_type = (_child_value(refn, "TYPE") or "").strip().casefold() + if refn_value and ("fsftid" in refn_type or "familysearch" in refn_type): + return refn_value + return None + + +def _person_name(element: Element) -> str: + raw_name = _child_value(element, "NAME") + if raw_name is None: + return "Unknown" + cleaned = raw_name.replace("/", " ").strip() + return " ".join(part for part in cleaned.split() if part) or "Unknown" + + +def _parse_exact_date(raw: str | None) -> date | None: + if raw is None: + return None + tokens = [token for token in raw.strip().upper().split() if token] + if len(tokens) != 3: + return None + day_token, month_token, year_token = tokens + if month_token not in _MONTHS: + return None + try: + return date(year=int(year_token), month=_MONTHS[month_token], day=int(day_token)) + except ValueError: + return None + + +def _first_child(element: Element | None, tag: str) -> Element | None: + if element is None: + return None + for child in element.get_child_elements(): + if child.get_tag() == tag: + return child + return None + + +def _children(element: Element | None, tag: str) -> list[Element]: + if element is None: + return [] + return [child for child in element.get_child_elements() if child.get_tag() == tag] + + +def _child_value(element: Element | None, tag: str) -> str | None: + child = _first_child(element, tag) + if child is None: + return None + value = (child.get_value() or "").strip() + return value or None + + +def _flatten_tag_values(element: Element) -> str | None: + lines: list[str] = [] + + def walk(node: Element) -> None: + value = (node.get_value() or "").strip() + if value: + lines.append(f"{node.get_tag()}: {value}") + for child in node.get_child_elements(): + walk(child) + + walk(element) + return " | ".join(lines) if lines else None diff --git a/src/transcription/services/maintenance.py b/src/transcription/services/maintenance.py index d564d9a..68cee91 100644 --- a/src/transcription/services/maintenance.py +++ b/src/transcription/services/maintenance.py @@ -8,6 +8,7 @@ from datetime import UTC from datetime import datetime from pathlib import Path from uuid import UUID +from uuid import uuid4 from sqlalchemy import update from sqlalchemy.exc import SQLAlchemyError @@ -26,6 +27,9 @@ from transcription.errors import ErrorCategory from transcription.errors import classify_unexpected_error from .base import ServiceBase +from .gedcom_import import GedcomImportError +from .gedcom_import import import_gedcom_file +from .media_storage import persist_named_media def _utc_now_naive() -> datetime: @@ -49,6 +53,38 @@ class MaintenanceError(AppError): class MaintenanceService(ServiceBase): """Persist and execute background maintenance runs.""" + async def store_gedcom_upload(self, *, filename: str, file_bytes: bytes) -> str: + if not file_bytes: + raise MaintenanceError( + "GEDCOM upload is empty.", + category=ErrorCategory.VALIDATION, + suggestion="Upload a non-empty .ged file and retry.", + ) + if Path(filename).suffix.casefold() != ".ged": + raise MaintenanceError( + "Unsupported GEDCOM upload format.", + category=ErrorCategory.USER_INPUT, + suggestion="Upload a file with a .ged extension.", + ) + stored_path = await persist_named_media( + root=self.settings.upload_dir, + namespace=Path("genealogy"), + filename=filename, + file_bytes=file_bytes, + filename_stem=str(uuid4()), + error=MaintenanceError, + failure_message="GEDCOM file could not be persisted.", + failure_suggestion="Check upload directory permissions and retry.", + log_label="gedcom file", + ) + return str(stored_path.resolve().relative_to(self.settings.upload_dir.resolve()).as_posix()) + + def latest_gedcom_upload_path(self) -> str | None: + latest = self._latest_gedcom_upload() + if latest is None: + return None + return str(latest.resolve().relative_to(self.settings.upload_dir.resolve()).as_posix()) + async def list_runs( self, *, @@ -174,6 +210,8 @@ class MaintenanceService(ServiceBase): return await self._execute_backup() if run.job_type == MaintenanceJobType.STORAGE_RECONCILIATION: return await self._execute_storage_reconciliation() + if run.job_type == MaintenanceJobType.GEDCOM_IMPORT: + return await self._execute_gedcom_import() raise MaintenanceError( "Unsupported maintenance job type.", category=ErrorCategory.VALIDATION, @@ -247,6 +285,55 @@ class MaintenanceService(ServiceBase): output="No storage reconciliation mismatches detected.", ) + async def _execute_gedcom_import(self) -> MaintenanceExecution: + latest_upload = self._latest_gedcom_upload() + if latest_upload is None: + return MaintenanceExecution( + status=MaintenanceRunStatus.FAILED, + summary="No GEDCOM upload is available.", + output="No .ged file found under uploads/genealogy.", + error_detail="Missing GEDCOM upload in uploads/genealogy.", + ) + + try: + async with self._session_scope() as session: + result = await import_gedcom_file(session=session, file_path=latest_upload) + except GedcomImportError as exc: + return MaintenanceExecution( + status=MaintenanceRunStatus.FAILED, + summary="GEDCOM import failed.", + output=f"GEDCOM import failed for {latest_upload.name}.", + error_detail=exc.detail, + ) + except Exception as exc: # noqa: BLE001 + error = classify_unexpected_error(exc, operation="maintenance.gedcom_import") + return MaintenanceExecution( + status=MaintenanceRunStatus.FAILED, + summary="GEDCOM import failed.", + output=f"GEDCOM import failed for {latest_upload.name}.", + error_detail=error.detail, + ) + + summary = ( + f"Imported {result.new_people + result.updated_people} people " + f"({result.new_people} new, {result.updated_people} updated) and " + f"{result.new_families + result.updated_families} families " + f"({result.new_families} new, {result.updated_families} updated)." + ) + output_lines = [ + f"GEDCOM file: {latest_upload.as_posix()}", + summary, + f"Skipped people without FamilySearch ID: {result.skipped_people_without_fs_id}", + f"Skipped families without FamilySearch ID: {result.skipped_families_without_fs_id}", + f"Family child links written: {result.family_children}", + f"Imported citations: {result.citations}", + ] + return MaintenanceExecution( + status=MaintenanceRunStatus.SUCCEEDED, + summary=summary, + output="\n".join(output_lines), + ) + async def _collect_storage_mismatches(self) -> list[str]: upload_root = self.settings.upload_dir folder_ids = _document_folder_ids(upload_root) @@ -282,6 +369,15 @@ class MaintenanceService(ServiceBase): ) return mismatches + def _latest_gedcom_upload(self) -> Path | None: + genealogy_root = self.settings.upload_dir / "genealogy" + if not genealogy_root.exists(): + return None + candidates = [path for path in genealogy_root.rglob("*.ged") if path.is_file()] + if not candidates: + return None + return max(candidates, key=lambda path: (path.stat().st_mtime_ns, path.name.casefold())) + async def _document_ids(self) -> set[str]: async with self._session_scope() as session: rows = await session.exec(select(Document.id)) diff --git a/src/transcription/ui/components/upload_panel.py b/src/transcription/ui/components/upload_panel.py index fd0c25c..299ead1 100644 --- a/src/transcription/ui/components/upload_panel.py +++ b/src/transcription/ui/components/upload_panel.py @@ -12,6 +12,7 @@ from nicegui import ui from transcription.services.source_media import SOURCE_EXTENSIONS SOURCE_UPLOAD_EXTENSIONS: tuple[str, ...] = tuple(sorted(SOURCE_EXTENSIONS)) +GEDCOM_UPLOAD_EXTENSIONS: tuple[str, ...] = (".ged",) IMAGE_UPLOAD_EXTENSIONS: tuple[str, ...] = ( ".bmp", ".gif", diff --git a/src/transcription/ui/pages/settings_page.py b/src/transcription/ui/pages/settings_page.py index 891ea69..58afb9f 100644 --- a/src/transcription/ui/pages/settings_page.py +++ b/src/transcription/ui/pages/settings_page.py @@ -8,6 +8,7 @@ from typing import Any from uuid import UUID from fastapi import Request +from nicegui import events from nicegui import ui from transcription.config import Settings @@ -27,6 +28,8 @@ from transcription.ui.components.primitives import render_empty_state from transcription.ui.components.primitives import section_header_row from transcription.ui.components.table.common import build_table from transcription.ui.components.table.registry import render_registry_table +from transcription.ui.components.upload_panel import GEDCOM_UPLOAD_EXTENSIONS +from transcription.ui.components.upload_panel import render_upload_picker from transcription.ui.homepage_store import read_homepage_markdown from transcription.ui.homepage_store import save_homepage_markdown from transcription.ui.runtime_settings_store import HIDDEN_SETTINGS_CATEGORIES @@ -516,6 +519,11 @@ def register_page(*, settings: Settings) -> None: # noqa: PLR0915 ui.label( "Queue maintenance tasks for worker execution. Runs are persisted with summary and logs." ).classes("text-xs ui-text-muted mb-3") + await _render_gedcom_import_controls( + maintenance=maintenance, + request=request, + refresh=render_maintenance.refresh, + ) with ui.row().classes("w-full items-center gap-2"): ui.button( "Run Backup", @@ -526,7 +534,7 @@ def register_page(*, settings: Settings) -> None: # noqa: PLR0915 request=request, refresh=render_maintenance.refresh, ), - ).classes("ui-btn-primary") + ).props("flat") ui.button( "Run Storage Reconciliation", icon="rule", @@ -652,14 +660,7 @@ def register_page(*, settings: Settings) -> None: # noqa: PLR0915 ui.button("View Log", icon="visibility", on_click=view_log).props("flat") ui.button("Download Log", icon="download", on_click=download_log).props("flat") - active_statuses = frozenset( - { - MaintenanceRunStatus.QUEUED, - MaintenanceRunStatus.PROCESSING, - } - ) - if any(run.status in active_statuses for run in runs): - ui.timer(4.0, render_maintenance.refresh, once=True) + _schedule_maintenance_refresh_if_active(runs=runs, refresh=render_maintenance.refresh) @ui.refreshable async def render_runtime_settings() -> None: @@ -842,6 +843,76 @@ async def _enqueue_maintenance_run( refresh() +async def _render_gedcom_import_controls( + *, + maintenance: MaintenanceService, + request: Request, + refresh, +) -> None: + ui.label("GEDCOM Import").classes("text-sm font-semibold") + ui.label("Upload a FamilySearch GEDCOM export (.ged), then queue a GEDCOM import run.").classes( + "text-xs ui-text-muted mb-2" + ) + + async def on_gedcom_upload(event: events.UploadEventArguments) -> None: + payload = await event.file.read() + upload_outcome = await run_ui_action( + operation="settings.maintenance.gedcom.upload", + title="GEDCOM upload failed", + action=lambda: maintenance.store_gedcom_upload( + filename=event.file.name, + file_bytes=payload, + ), + ) + if not upload_outcome.ok or upload_outcome.value is None: + return + ui.notify(f"Uploaded GEDCOM file: {event.file.name}", type="positive") + refresh() + + render_upload_picker( + on_upload=on_gedcom_upload, + label="Upload GEDCOM file", + extensions=GEDCOM_UPLOAD_EXTENSIONS, + ) + + latest_gedcom_outcome = await run_ui_action( + operation="settings.maintenance.gedcom.latest", + title="GEDCOM uploads unavailable", + action=lambda: _latest_gedcom_upload_path(maintenance=maintenance), + ) + latest_gedcom_path = latest_gedcom_outcome.value if latest_gedcom_outcome.ok else None + ui.label( + f"Latest GEDCOM upload: {latest_gedcom_path}" if latest_gedcom_path else "Latest GEDCOM upload: none" + ).classes("text-xs ui-text-muted mb-2") + + with ui.row().classes("w-full items-center gap-2"): + ui.button( + "Run GEDCOM Import", + icon="upload_file", + on_click=lambda: _enqueue_maintenance_run( + maintenance=maintenance, + job_type=MaintenanceJobType.GEDCOM_IMPORT, + request=request, + refresh=refresh, + ), + ).classes("ui-btn-primary") + + +async def _latest_gedcom_upload_path(*, maintenance: MaintenanceService) -> str | None: + return maintenance.latest_gedcom_upload_path() + + +def _schedule_maintenance_refresh_if_active(*, runs: list[Any], refresh) -> None: + active_statuses = frozenset( + { + MaintenanceRunStatus.QUEUED, + MaintenanceRunStatus.PROCESSING, + } + ) + if any(run.status in active_statuses for run in runs): + ui.timer(4.0, refresh, once=True) + + def _format_timestamp(value: datetime | None) -> str: if value is None: return "-" diff --git a/tests/fixtures/gedcom/sample_familysearch.ged b/tests/fixtures/gedcom/sample_familysearch.ged new file mode 100644 index 0000000..3c562b9 --- /dev/null +++ b/tests/fixtures/gedcom/sample_familysearch.ged @@ -0,0 +1,42 @@ +0 HEAD +1 SOUR getmyancestors +1 GEDC +2 VERS 5.5.1 +1 CHAR UTF-8 +0 @I1@ INDI +1 NAME John /Doe/ +1 REFN KWC1-ABC +2 TYPE FSFTID +1 BIRT +2 DATE 1 JAN 1900 +2 PLAC Springfield, Illinois +2 SOUR Birth Register +3 PAGE p. 12 +1 DEAT +2 DATE 5 FEB 1970 +2 PLAC Shelbyville, Illinois +2 SOUR Death Certificate +3 TEXT County archive +0 @I2@ INDI +1 NAME Jane /Smith/ +1 _FSFTID LMN2-XYZ +1 BIRT +2 DATE 12 MAR 1905 +2 PLAC Capital City, Illinois +0 @I3@ INDI +1 NAME Child /Doe/ +1 REFN CHD3-123 +2 TYPE FSFTID +0 @F1@ FAM +1 REFN FAM-001 +2 TYPE FSFTID +1 HUSB @I1@ +1 WIFE @I2@ +1 CHIL @I3@ +2 PEDI adopted +1 MARR +2 DATE 4 APR 1925 +2 PLAC Springfield, Illinois +2 SOUR Marriage License +3 PAGE Book 9 +0 TRLR diff --git a/tests/services/test_gedcom_import.py b/tests/services/test_gedcom_import.py new file mode 100644 index 0000000..4d456e0 --- /dev/null +++ b/tests/services/test_gedcom_import.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from datetime import date +from pathlib import Path + +import pytest +from sqlmodel import select + +from transcription.db.models import GenealogyCitation +from transcription.db.models import GenealogyFamily +from transcription.db.models import GenealogyFamilyChild +from transcription.db.models import GenealogyPerson +from transcription.services.gedcom_import import import_gedcom_file +from transcription.services.gedcom_import import parse_gedcom + +_FIXTURE_PATH = Path(__file__).resolve().parents[1] / "fixtures" / "gedcom" / "sample_familysearch.ged" + + +def test_parse_gedcom_extracts_people_families_and_citations(): + parsed = parse_gedcom(file_path=_FIXTURE_PATH) + + assert len(parsed.people) == 3 + john = next(person for person in parsed.people if person.fs_id == "KWC1-ABC") + assert john.full_name == "John Doe" + assert john.birth_date == date(1900, 1, 1) + assert john.birth_place == "Springfield, Illinois" + assert john.death_date == date(1970, 2, 5) + assert len(john.citations) == 2 + assert "Birth Register" in john.citations[0].raw_citation_text + + assert len(parsed.families) == 1 + family = parsed.families[0] + assert family.fs_family_id == "FAM-001" + assert family.marriage_date == date(1925, 4, 4) + assert family.marriage_place == "Springfield, Illinois" + assert len(family.children) == 1 + assert family.children[0].relationship_type == "adopted" + assert len(family.citations) == 1 + assert "Marriage License" in family.citations[0].raw_citation_text + + +@pytest.mark.asyncio +async def test_import_gedcom_file_is_idempotent(default_session_factory): + async with default_session_factory() as session: + first = await import_gedcom_file(session=session, file_path=_FIXTURE_PATH) + async with default_session_factory() as session: + second = await import_gedcom_file(session=session, file_path=_FIXTURE_PATH) + async with default_session_factory() as session: + people = (await session.exec(select(GenealogyPerson))).all() + families = (await session.exec(select(GenealogyFamily))).all() + children = (await session.exec(select(GenealogyFamilyChild))).all() + citations = (await session.exec(select(GenealogyCitation))).all() + + assert first.new_people == 3 + assert first.new_families == 1 + assert first.family_children == 1 + assert first.citations == 3 + + assert second.new_people == 0 + assert second.updated_people == 0 + assert second.new_families == 0 + assert second.updated_families == 0 + assert second.family_children == 1 + assert second.citations == 3 + + assert len(people) == 3 + assert len(families) == 1 + assert len(children) == 1 + assert len(citations) == 3 diff --git a/tests/services/test_maintenance_service.py b/tests/services/test_maintenance_service.py index 5e69ea5..017aff5 100644 --- a/tests/services/test_maintenance_service.py +++ b/tests/services/test_maintenance_service.py @@ -22,11 +22,13 @@ async def test_enqueue_and_list_runs(default_session_factory, default_settings): first = await service.enqueue_run(job_type=MaintenanceJobType.BACKUP, triggered_by="test") second = await service.enqueue_run(job_type=MaintenanceJobType.STORAGE_RECONCILIATION, triggered_by="test") + third = await service.enqueue_run(job_type=MaintenanceJobType.GEDCOM_IMPORT, triggered_by="test") runs = await service.list_runs(limit=10) - assert len(runs) == 2 - assert runs[0].id == second.id - assert runs[1].id == first.id + assert len(runs) == 3 + assert runs[0].id == third.id + assert runs[1].id == second.id + assert runs[2].id == first.id assert runs[0].status == MaintenanceRunStatus.QUEUED @@ -72,3 +74,20 @@ async def test_list_runs_raises_when_table_is_missing(default_session_factory, d with pytest.raises(MaintenanceError) as exc: await service.list_runs(limit=10) assert exc.value.category == ErrorCategory.INFRA_PERSISTENT + + +@pytest.mark.asyncio +async def test_execute_run_dispatches_gedcom_import(default_session_factory, default_settings, monkeypatch): + service = MaintenanceService(session_factory=default_session_factory, settings=default_settings) + run = await service.enqueue_run(job_type=MaintenanceJobType.GEDCOM_IMPORT, triggered_by="test") + + async def _fake_gedcom_import(): + return MaintenanceExecution( + status=MaintenanceRunStatus.SUCCEEDED, + summary="GEDCOM imported", + output="ok", + ) + + monkeypatch.setattr(service, "_execute_gedcom_import", _fake_gedcom_import) + execution = await service._execute_run(run) + assert execution.summary == "GEDCOM imported" diff --git a/tests/ui/test_pages_registration.py b/tests/ui/test_pages_registration.py index 2f68ee2..5e0e6d8 100644 --- a/tests/ui/test_pages_registration.py +++ b/tests/ui/test_pages_registration.py @@ -29,5 +29,6 @@ class TestPageRegistration: assert "Prompts" in settings_response.text assert "Home Page Text" in settings_response.text assert "Maintenance" in settings_response.text + assert "Run GEDCOM Import" in settings_response.text assert "Other settings not shown here" in settings_response.text assert "README.md" not in settings_response.text diff --git a/uv.lock b/uv.lock index f08be04..cf6485e 100644 --- a/uv.lock +++ b/uv.lock @@ -1860,6 +1860,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/96/82f6328e410515fab21d5602ba35b9377a47b5a141a0c1f9efa00ce21eb4/python_engineio-4.13.3-py3-none-any.whl", hash = "sha256:1f60ecaf1358190f0e26c48c578a60428dc02a8f1295bc3dbf53d1b31116821f", size = 59993 }, ] +[[package]] +name = "python-gedcom" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/79/f2cb205a79b4995df74f86a0f7b3b278e4910b7a337c6fa26c5f88067117/python_gedcom-1.1.0.tar.gz", hash = "sha256:18a6d8b1832e9c0e912cbfa7958941544a88d1447e753d3aaa1f9f3c5dab1e38", size = 168754 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/41/90707f85cf2585f1240a773793c0deff14b22452575c582cc0b8c8bde551/python_gedcom-1.1.0-py3-none-any.whl", hash = "sha256:98ce4f4da3949727c235e0adbe6b96360fd4840a6ccf3c8b20cab3407041f65f", size = 32730 }, +] + [[package]] name = "python-multipart" version = "0.0.32" @@ -2169,6 +2178,7 @@ dependencies = [ { name = "psycopg2-binary" }, { name = "pydantic" }, { name = "pydantic-settings" }, + { name = "python-gedcom" }, { name = "sqlmodel" }, ] @@ -2196,6 +2206,7 @@ requires-dist = [ { name = "psycopg2-binary", specifier = ">=2.9.12" }, { name = "pydantic", specifier = ">=2.13.4" }, { name = "pydantic-settings", specifier = ">=2.9.1" }, + { name = "python-gedcom", specifier = ">=1.1.0" }, { name = "sqlmodel", specifier = ">=0.0.25" }, ]