generated from john/python-template
Misc updates I don't remember making
This commit is contained in:
Jim Lancaster
@@ -0,0 +1 @@
|
||||
GEMINI_API_KEY=AQ.Ab8RN6Jd5iE56jbjLSa4SJhO01pfR6MOctNpokJHp0uz5TubQA
|
||||
+508490
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,235 @@
|
||||
# extract_fs_links.py
|
||||
# Extracts FamilySearch links and IDs from PDF files.
|
||||
# 1. In FamilySearch.org: select a Person -> View tree -> Options -> Print -> Save as PDF
|
||||
# This will provide you with a tree segment that 3 levels above and one below the selected person.
|
||||
# 2. Select a person on the top row and repeat step 1. Do this for as many top-level people as needed.
|
||||
# 3. Run this script to extract the FamilySearch links and IDs from the saved PDF files.
|
||||
|
||||
import csv
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unicodedata
|
||||
from xmlrpc import client
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
import pandas as pd
|
||||
import pdfplumber
|
||||
import pypdfium2 as pdfium
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Configuration
|
||||
SOURCE_DIR = Path(r"C:\Proton Drive\bbchops\My files\Archive\FamilySearch")
|
||||
OUTPUT_FILE = SOURCE_DIR / "combined_pedigree_with_ids.csv"
|
||||
|
||||
FS_ID_PATTERN = re.compile(r"([A-Z0-9]{4}-[A-Z0-9]{3,4})")
|
||||
|
||||
|
||||
# --- Schema Definition for Structured Vision Extraction ---
|
||||
class PersonNode(BaseModel):
|
||||
person_name: str = Field(
|
||||
description="Full name of the person (excluding dates or badge titles)"
|
||||
)
|
||||
person_dates: str = Field(
|
||||
description="Date range (e.g. 1890-1975, 1795-Deceased, or blank)"
|
||||
)
|
||||
parent1_name: str = Field(
|
||||
default="", description="Full name of Father/Parent 1 if visible"
|
||||
)
|
||||
parent1_dates: str = Field(
|
||||
default="", description="Date range of Father/Parent 1 if visible"
|
||||
)
|
||||
parent2_name: str = Field(
|
||||
default="", description="Full name of Mother/Parent 2 if visible"
|
||||
)
|
||||
parent2_dates: str = Field(
|
||||
default="", description="Date range of Mother/Parent 2 if visible"
|
||||
)
|
||||
|
||||
|
||||
class PedigreeTree(BaseModel):
|
||||
records: list[PersonNode]
|
||||
|
||||
|
||||
# --- Helper Functions ---
|
||||
def clean_name(text: str) -> str:
|
||||
"""Normalizes string and strips dates/artifacts for reliable matching."""
|
||||
if not text:
|
||||
return ""
|
||||
text = unicodedata.normalize("NFKD", text)
|
||||
text = re.sub(r"[\u2010-\u2015\u2212\uFE58\uFE63\uFF0D]", "-", text)
|
||||
for badge in [
|
||||
"MAYFLOWER",
|
||||
"DESCENDANT",
|
||||
"PATRIOT",
|
||||
"DEAD",
|
||||
"END",
|
||||
"FamilySearch",
|
||||
]:
|
||||
text = re.sub(rf"\b{badge}\b", "", text, flags=re.IGNORECASE)
|
||||
text = re.sub(r"\b\d{4}\s*-\s*(?:\d{4}|Deceased)\b", "", text)
|
||||
return " ".join(text.split()).strip().lower()
|
||||
|
||||
|
||||
def clean_dates(text: str) -> str:
|
||||
"""Standardizes date ranges to 'YYYY-YYYY' or 'YYYY-Deceased'."""
|
||||
if not text:
|
||||
return ""
|
||||
text = unicodedata.normalize("NFKD", text)
|
||||
text = re.sub(r"[\u2010-\u2015\u2212\uFE58\uFE63\uFF0D]", "-", text)
|
||||
match = re.search(r"(\b\d{4}\s*-\s*(?:\d{4}|Deceased)\b)", text)
|
||||
return match.group(1).replace(" ", "") if match else text.strip()
|
||||
|
||||
|
||||
# --- PASS 1: Link & ID Extraction ---
|
||||
def extract_link_dataframe(source_dir: Path) -> pd.DataFrame:
|
||||
rows = []
|
||||
pdf_files = sorted(source_dir.glob("*.pdf"))
|
||||
|
||||
for pdf_path in pdf_files:
|
||||
with pdfplumber.open(str(pdf_path)) as pdf:
|
||||
for page in pdf.pages:
|
||||
hyperlinks = []
|
||||
for annot in page.hyperlinks:
|
||||
uri = annot.get("uri", "")
|
||||
match = FS_ID_PATTERN.search(uri)
|
||||
if match:
|
||||
hyperlinks.append(
|
||||
{
|
||||
"fs_id": match.group(1),
|
||||
"x0": annot.get("x0", 0),
|
||||
"top": annot.get("top", 0),
|
||||
"x1": annot.get("x1", 0),
|
||||
"bottom": annot.get("bottom", 0),
|
||||
}
|
||||
)
|
||||
|
||||
words = page.extract_words(x_tolerance=2, y_tolerance=2)
|
||||
for link in hyperlinks:
|
||||
lx_center = (link["x0"] + link["x1"]) / 2
|
||||
matched_words = [
|
||||
w
|
||||
for w in words
|
||||
if abs(((w["x0"] + w["x1"]) / 2) - lx_center) <= 28
|
||||
and (link["top"] - 5) <= w["top"] <= (link["bottom"] + 75)
|
||||
]
|
||||
matched_words.sort(key=lambda w: (w["top"], w["x0"]))
|
||||
block_text = " ".join(w["text"] for w in matched_words)
|
||||
|
||||
rows.append(
|
||||
{
|
||||
"match_key": clean_name(block_text),
|
||||
"Date_Range": clean_dates(block_text),
|
||||
"FS_ID": link["fs_id"],
|
||||
}
|
||||
)
|
||||
|
||||
df_links = pd.DataFrame(rows)
|
||||
# Deduplicate in case multiple links resolve to the same person key
|
||||
df_links = df_links.drop_duplicates(subset=["match_key"])
|
||||
return df_links
|
||||
|
||||
|
||||
# --- PASS 2: Vision OCR Pass with Gemini 2.5 Flash ---
|
||||
def extract_vision_dataframe(source_dir: Path) -> pd.DataFrame:
|
||||
client = genai.Client()
|
||||
all_nodes = []
|
||||
pdf_files = sorted(source_dir.glob("*.pdf"))
|
||||
|
||||
for pdf_path in pdf_files:
|
||||
print(f"Running Vision OCR on {pdf_path.name}...")
|
||||
# Render PDF page to high-res image in-memory
|
||||
pdf_doc = pdfium.PdfDocument(str(pdf_path))
|
||||
page = pdf_doc[0]
|
||||
image = page.render(scale=2.0).to_pil()
|
||||
|
||||
img_bytes = io.BytesIO()
|
||||
image.save(img_bytes, format="PNG")
|
||||
|
||||
prompt = (
|
||||
"Analyze this family tree chart. Extract all individuals shown as nodes along with "
|
||||
"their birth-death date ranges and their direct parents (Parent 1 and Parent 2) "
|
||||
"as indicated by the tree hierarchy lines. Strip out decorative badges like 'Mayflower Descendant'."
|
||||
)
|
||||
|
||||
response = client.models.generate_content(
|
||||
model="gemini-3.6-flash", # If unavailable on your tier, use "gemini-2.0-flash" or "gemini-1.5-flash"
|
||||
contents=[
|
||||
types.Part.from_bytes(data=img_bytes.getvalue(), mime_type="image/png"),
|
||||
prompt,
|
||||
],
|
||||
config=types.GenerateContentConfig(
|
||||
response_mime_type="application/json",
|
||||
response_schema=PedigreeTree,
|
||||
temperature=0.0,
|
||||
),
|
||||
)
|
||||
|
||||
tree_data: PedigreeTree = response.parsed
|
||||
for item in tree_data.records:
|
||||
all_nodes.append(
|
||||
{
|
||||
"Person": item.person_name.strip(),
|
||||
"Person_Dates": clean_dates(item.person_dates),
|
||||
"Parent1": item.parent1_name.strip(),
|
||||
"Parent1_Dates": clean_dates(item.parent1_dates),
|
||||
"Parent2": item.parent2_name.strip(),
|
||||
"Parent2_Dates": clean_dates(item.parent2_dates),
|
||||
}
|
||||
)
|
||||
|
||||
return pd.DataFrame(all_nodes).drop_duplicates(subset=["Person", "Person_Dates"])
|
||||
|
||||
|
||||
# --- PASS 3: Merging and Populating IDs ---
|
||||
def build_combined_pedigree():
|
||||
print("Pass 1: Extracting URLs and bounding boxes...")
|
||||
df_links = extract_link_dataframe(SOURCE_DIR)
|
||||
|
||||
print("Pass 2: Running Gemini Vision OCR on pedigree charts...")
|
||||
df_pedigree = extract_vision_dataframe(SOURCE_DIR)
|
||||
|
||||
print("Pass 3: Joining IDs into Pedigree structure...")
|
||||
# Lookup dictionary mapping cleaned name -> FS ID
|
||||
id_lookup = dict(zip(df_links["match_key"], df_links["FS_ID"]))
|
||||
|
||||
def get_id(name):
|
||||
return id_lookup.get(clean_name(name), "")
|
||||
|
||||
df_pedigree["Person_FS_ID"] = df_pedigree["Person"].apply(get_id)
|
||||
df_pedigree["Parent1_FS_ID"] = df_pedigree["Parent1"].apply(get_id)
|
||||
df_pedigree["Parent2_FS_ID"] = df_pedigree["Parent2"].apply(get_id)
|
||||
|
||||
# Reorder into the requested final column layout
|
||||
final_df = df_pedigree[
|
||||
[
|
||||
"Person",
|
||||
"Person_Dates",
|
||||
"Person_FS_ID",
|
||||
"Parent1",
|
||||
"Parent1_Dates",
|
||||
"Parent1_FS_ID",
|
||||
"Parent2",
|
||||
"Parent2_Dates",
|
||||
"Parent2_FS_ID",
|
||||
]
|
||||
].rename(
|
||||
columns={
|
||||
"Person_Dates": "Date Range",
|
||||
"Person_FS_ID": "FS ID",
|
||||
"Parent1": "Parent 1",
|
||||
"Parent1_Dates": "Date Range",
|
||||
"Parent1_FS_ID": "FS ID",
|
||||
"Parent2": "Parent 2",
|
||||
"Parent2_Dates": "Date Range",
|
||||
"Parent2_FS_ID": "FS ID",
|
||||
}
|
||||
)
|
||||
|
||||
final_df.to_csv(OUTPUT_FILE, index=False, encoding="utf-8-sig")
|
||||
print(f"\nCompleted! Combined table exported to:\n{OUTPUT_FILE}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
build_combined_pedigree()
|
||||
@@ -2,8 +2,8 @@ import os
|
||||
import fitz # PyMuPDF
|
||||
|
||||
# Configuration
|
||||
PDF_PATH = "C:/Proton Drive/bbchops/My files/Archive/HigServiceRecords.pdf"
|
||||
OUTPUT_DIR = "C:/Proton Drive/bbchops/My files/Archive/HigServiceRecords"
|
||||
PDF_PATH = "C:/Proton Drive/bbchops/My files/Archive/Cochran/Cochran bios/Effie Ruth Cochran Montague.pdf"
|
||||
OUTPUT_DIR = "C:/Proton Drive/bbchops/My files/Archive/Cochran/Cochran bios/test"
|
||||
|
||||
|
||||
def extract_and_convert_images(pdf_path, output_dir):
|
||||
@@ -0,0 +1,166 @@
|
||||
import csv
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unicodedata
|
||||
import pdfplumber
|
||||
|
||||
# Configuration
|
||||
SOURCE_DIR = Path(r"C:\Proton Drive\bbchops\My files\Archive\FamilySearch")
|
||||
OUTPUT_FILE = SOURCE_DIR / "familysearch_people_with_links.csv"
|
||||
|
||||
# Regex pattern for FamilySearch Person IDs
|
||||
FS_ID_PATTERN = re.compile(r"([A-Z0-9]{4}-[A-Z0-9]{3,4})")
|
||||
|
||||
# Badge/Artifact labels to strip
|
||||
BADGE_LABELS = {
|
||||
"MAYFLOWER",
|
||||
"DESCENDANT",
|
||||
"PATRIOT",
|
||||
"DEAD",
|
||||
"END",
|
||||
"FAMILYSEARCH",
|
||||
}
|
||||
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
"""Normalizes dashes, unicode artifacts, and collapses whitespace."""
|
||||
if not text:
|
||||
return ""
|
||||
# Replace non-breaking spaces, en-dashes, em-dashes, and special hyphens
|
||||
text = unicodedata.normalize("NFKD", text)
|
||||
text = re.sub(r"[\u2010-\u2015\u2212\uFE58\uFE63\uFF0D]", "-", text)
|
||||
return " ".join(text.split()).strip()
|
||||
|
||||
|
||||
def extract_people_from_pdf(pdf_path: Path):
|
||||
extracted_records = []
|
||||
|
||||
try:
|
||||
with pdfplumber.open(str(pdf_path)) as pdf:
|
||||
for page in pdf.pages:
|
||||
hyperlinks = []
|
||||
for annot in page.hyperlinks:
|
||||
uri = annot.get("uri", "")
|
||||
match = FS_ID_PATTERN.search(uri)
|
||||
if match:
|
||||
hyperlinks.append(
|
||||
{
|
||||
"fs_id": match.group(1),
|
||||
"uri": uri,
|
||||
"x0": annot.get("x0", 0),
|
||||
"top": annot.get("top", 0),
|
||||
"x1": annot.get("x1", 0),
|
||||
"bottom": annot.get("bottom", 0),
|
||||
}
|
||||
)
|
||||
|
||||
words = page.extract_words(
|
||||
keep_blank_chars=False, x_tolerance=2, y_tolerance=2
|
||||
)
|
||||
|
||||
for link in hyperlinks:
|
||||
lx_center = (link["x0"] + link["x1"]) / 2
|
||||
|
||||
# Narrow horizontal span (+/- 28px) to isolate this specific node column
|
||||
# Search from link top down to +75px
|
||||
matched_words = [
|
||||
w
|
||||
for w in words
|
||||
if abs(((w["x0"] + w["x1"]) / 2) - lx_center) <= 28
|
||||
and (link["top"] - 5) <= w["top"] <= (link["bottom"] + 75)
|
||||
]
|
||||
|
||||
if not matched_words:
|
||||
continue
|
||||
|
||||
# Group words into lines by vertical position
|
||||
matched_words.sort(key=lambda w: (w["top"], w["x0"]))
|
||||
lines = []
|
||||
current_line = []
|
||||
current_y = None
|
||||
|
||||
for w in matched_words:
|
||||
if current_y is None or abs(w["top"] - current_y) <= 4:
|
||||
current_line.append(w["text"])
|
||||
current_y = w["top"]
|
||||
else:
|
||||
lines.append(" ".join(current_line))
|
||||
current_line = [w["text"]]
|
||||
current_y = w["top"]
|
||||
if current_line:
|
||||
lines.append(" ".join(current_line))
|
||||
|
||||
# Parse dates and clean names line-by-line
|
||||
name_parts = []
|
||||
date_range = ""
|
||||
|
||||
for line in lines:
|
||||
cleaned_line = clean_text(line)
|
||||
|
||||
# Filter out badge text
|
||||
if (
|
||||
cleaned_line.upper() in BADGE_LABELS
|
||||
or "MAYFLOWER" in cleaned_line.upper()
|
||||
):
|
||||
continue
|
||||
|
||||
# Check for Date Range: e.g. 1890-1975 or 1715-Deceased
|
||||
date_match = re.search(
|
||||
r"(\b\d{4}\s*-\s*(?:\d{4}|Deceased)\b)",
|
||||
cleaned_line,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if date_match:
|
||||
date_range = date_match.group(1).replace(" ", "")
|
||||
# If text exists before the date on the same line, retain it
|
||||
prefix = cleaned_line[: date_match.start()].strip()
|
||||
if prefix:
|
||||
name_parts.append(prefix)
|
||||
else:
|
||||
name_parts.append(cleaned_line)
|
||||
|
||||
person_name = clean_text(" ".join(name_parts))
|
||||
|
||||
extracted_records.append(
|
||||
{
|
||||
"File": pdf_path.name,
|
||||
"Name": person_name,
|
||||
"Date_Range": date_range,
|
||||
"FS_ID": link["fs_id"],
|
||||
"URL": link["uri"],
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as err:
|
||||
print(f"[ERROR] Failed reading {pdf_path.name}: {err}")
|
||||
|
||||
return extracted_records
|
||||
|
||||
|
||||
def batch_extract(source_dir: Path, output_csv: Path):
|
||||
pdf_files = sorted(source_dir.glob("*.pdf"))
|
||||
if not pdf_files:
|
||||
print(f"No PDF files found in: {source_dir}")
|
||||
return
|
||||
|
||||
print(f"Processing {len(pdf_files)} PDF(s)...")
|
||||
|
||||
all_records = []
|
||||
for pdf_path in pdf_files:
|
||||
records = extract_people_from_pdf(pdf_path)
|
||||
all_records.extend(records)
|
||||
print(f" - {pdf_path.name}: {len(records)} person/link pair(s)")
|
||||
|
||||
output_csv.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Use utf-8-sig so Excel automatically reads hyphens and Unicode without Mojibake
|
||||
with open(output_csv, "w", newline="", encoding="utf-8-sig") as f:
|
||||
fieldnames = ["File", "Name", "Date_Range", "FS_ID", "URL"]
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(all_records)
|
||||
|
||||
print(f"\nDone. Saved to:\n{output_csv}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
batch_extract(SOURCE_DIR, OUTPUT_FILE)
|
||||
@@ -4,5 +4,14 @@ version = "0.1.0"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"copier>=9.15.1",
|
||||
"ged4py>=0.5.2",
|
||||
"getmyancestors>=1.2.0",
|
||||
"google-genai>=2.20.0",
|
||||
"pandas>=3.0.5",
|
||||
"pdfplumber>=0.11.10",
|
||||
"pillow>=12.3.0",
|
||||
"pydantic>=2.13.4",
|
||||
"pymupdf>=1.28.0",
|
||||
"pypdf>=6.16.2",
|
||||
"pypdfium2>=5.13.0",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user